From aaad2a2693146ffcc9d3f16333c1457a827eceaf Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:01:18 +0200 Subject: [PATCH 1/7] initial draft --- AGENTS.md | 10 + docs/architecture/blueprints/README.md | 41 + docs/architecture/blueprints/concepts.md | 216 +++ .../decisions/001-standalone-feature.md | 37 + .../decisions/002-plan-then-apply.md | 54 + .../decisions/003-managed-fields.md | 51 + .../decisions/004-processors-propose.md | 58 + .../blueprints/decisions/005-app-gateways.md | 55 + .../blueprints/decisions/006-variant-merge.md | 58 + .../blueprints/decisions/README.md | 16 + docs/architecture/blueprints/processors.md | 152 +++ docs/architecture/blueprints/reference.md | 583 +++++++++ .../01_onboard_with_blueprint.py | 79 ++ .../02_custom_processor_and_naming.py | 114 ++ .../07_blueprints/matrox-convertip.yml | 41 + docs/examples/README.md | 11 + docs/getting-started-guide/05_Blueprints.md | 188 +++ docs/getting-started-guide/README.md | 1 + poetry.lock | 4 +- pyproject.toml | 1 + .../apps/inventory/app/app.py | 8 + .../apps/inventory/inventory_api.py | 30 + .../blueprints/__init__.py | 128 ++ .../blueprints/engine.py | 959 ++++++++++++++ .../blueprints/errors.py | 103 ++ .../blueprints/inspect.py | 883 +++++++++++++ .../blueprints/inventory.py | 475 +++++++ .../blueprints/loader.py | 186 +++ .../blueprints/models.py | 688 ++++++++++ .../blueprints/naming.py | 426 ++++++ .../blueprints/processors/__init__.py | 297 +++++ .../blueprints/processors/matrox.py | 263 ++++ .../blueprints/resolution.py | 509 ++++++++ .../schemas/blueprint-v1.schema.json | 1159 +++++++++++++++++ tests/blueprints/__init__.py | 0 tests/blueprints/conftest.py | 365 ++++++ tests/blueprints/test_blueprint_inputs.py | 106 ++ tests/blueprints/test_engine.py | 947 ++++++++++++++ tests/blueprints/test_loader.py | 86 ++ tests/blueprints/test_naming.py | 125 ++ tests/blueprints/test_processors.py | 285 ++++ tests/blueprints/test_resolution.py | 241 ++++ 42 files changed, 10037 insertions(+), 2 deletions(-) create mode 100644 docs/architecture/blueprints/README.md create mode 100644 docs/architecture/blueprints/concepts.md create mode 100644 docs/architecture/blueprints/decisions/001-standalone-feature.md create mode 100644 docs/architecture/blueprints/decisions/002-plan-then-apply.md create mode 100644 docs/architecture/blueprints/decisions/003-managed-fields.md create mode 100644 docs/architecture/blueprints/decisions/004-processors-propose.md create mode 100644 docs/architecture/blueprints/decisions/005-app-gateways.md create mode 100644 docs/architecture/blueprints/decisions/006-variant-merge.md create mode 100644 docs/architecture/blueprints/decisions/README.md create mode 100644 docs/architecture/blueprints/processors.md create mode 100644 docs/architecture/blueprints/reference.md create mode 100644 docs/examples/07_blueprints/01_onboard_with_blueprint.py create mode 100644 docs/examples/07_blueprints/02_custom_processor_and_naming.py create mode 100644 docs/examples/07_blueprints/matrox-convertip.yml create mode 100644 docs/getting-started-guide/05_Blueprints.md create mode 100644 src/videoipath_automation_tool/blueprints/__init__.py create mode 100644 src/videoipath_automation_tool/blueprints/engine.py create mode 100644 src/videoipath_automation_tool/blueprints/errors.py create mode 100644 src/videoipath_automation_tool/blueprints/inspect.py create mode 100644 src/videoipath_automation_tool/blueprints/inventory.py create mode 100644 src/videoipath_automation_tool/blueprints/loader.py create mode 100644 src/videoipath_automation_tool/blueprints/models.py create mode 100644 src/videoipath_automation_tool/blueprints/naming.py create mode 100644 src/videoipath_automation_tool/blueprints/processors/__init__.py create mode 100644 src/videoipath_automation_tool/blueprints/processors/matrox.py create mode 100644 src/videoipath_automation_tool/blueprints/resolution.py create mode 100644 src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json create mode 100644 tests/blueprints/__init__.py create mode 100644 tests/blueprints/conftest.py create mode 100644 tests/blueprints/test_blueprint_inputs.py create mode 100644 tests/blueprints/test_engine.py create mode 100644 tests/blueprints/test_loader.py create mode 100644 tests/blueprints/test_naming.py create mode 100644 tests/blueprints/test_processors.py create mode 100644 tests/blueprints/test_resolution.py diff --git a/AGENTS.md b/AGENTS.md index af93963..15b7060 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,16 @@ Response models live in `connector/models/` as Pydantic models. - `domain/` — thin view objects that hold a back-reference to the snapshot for cross-entity lookups - `model/` — raw Pydantic models for the API response (`collector.py`, etc.) +### Blueprints (standalone) + +`blueprints/` is a standalone feature (`BlueprintEngine(app)`); `VideoIPathApp` neither imports nor exposes it. Design record: [`docs/architecture/blueprints/`](docs/architecture/blueprints/README.md). YAML parsing (PyYAML, a regular dependency) lives only in `loader.py`. +- `models.py` / `loader.py` / `resolution.py` — strict document models, safe YAML loading, variant merge + driver/parameter validation, JSON Schema +- `engine.py` — `BlueprintEngine`, read-only `plan()`, `BlueprintPlan.apply()` executor (phases, conflicts, partial results) +- `inventory.py` / `inspect.py` — the only modules that call `app.inventory` / `app.inspect` (managed-field diffs, fresh scoped reads, transaction/tag writes) +- `naming.py` — `Text` / `Field` / `Join` naming blocks; `processors/` — `VertexProcessor` contract, per-engine `ProcessorRegistry`, built-in Matrox processor +- `schemas/blueprint-v1.schema.json` is generated; `tests/blueprints/test_resolution.py` fails when it is stale. Regenerate with: + `poetry run python -c "import json, pathlib; from videoipath_automation_tool.blueprints import Blueprint, ProcessorRegistry; pathlib.Path('src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json').write_text(json.dumps(Blueprint.json_schema(ProcessorRegistry()), indent=2, sort_keys=True) + '\n')"` + ### Driver versioning Driver schemas (Pydantic models for device `custom_settings`) are auto-generated from the VideoIPath API's JSON schema and live under `apps/inventory/model/drivers.py`. Run `set-videoipath-version ` to regenerate them for a different server version. The CLI scripts are in `src/vipat_cli_scripts/`. diff --git a/docs/architecture/blueprints/README.md b/docs/architecture/blueprints/README.md new file mode 100644 index 0000000..8f29f8c --- /dev/null +++ b/docs/architecture/blueprints/README.md @@ -0,0 +1,41 @@ +# Blueprints — Architecture + +Design record for the blueprint feature in this package +(`src/videoipath_automation_tool/blueprints/`). + +A blueprint is a reusable description of one device type. `BlueprintEngine` combines +that description with the facts of one device, previews the changes, and writes +them through the existing Inventory and Inspect apps. `VideoIPathApp` does not +import or expose the feature. You construct the engine yourself. + +For a short introduction, start with the +[getting-started page](../../getting-started-guide/05_Blueprints.md). Runnable +scripts are in [docs/examples/07_blueprints](../../examples/07_blueprints/). +Offline tests live under `tests/blueprints/`. There is no live-server suite for +this feature yet. + +## Reading order + +1. **[reference.md](./reference.md)** — complete user reference: engine API, + `BlueprintDevice`, the YAML document, variants, loading, naming, apply phases, + errors, configuration, limits, and the public API index. +2. **[processors.md](./processors.md)** — the vertex processor contract and the + built-in Matrox ConvertIP processor. +3. **[concepts.md](./concepts.md)** — boundaries, module map, and the plan/apply flow. +4. **[decisions/](./decisions/)** — the choices that would be expensive to reverse. + Start with [the index](./decisions/README.md). + +The topology phase commits through Inspect's existing transaction. That write +model is recorded in +[Inspect ADR-004](../inspect-app/decisions/004-commit-write-model.md). + +## Decision log + +| Question | Decision | Status | +| -------- | -------- | ------ | +| Where does the feature sit in the package? | [ADR-001](./decisions/001-standalone-feature.md) | Accepted | +| How is a change previewed and written? | [ADR-002](./decisions/002-plan-then-apply.md) | Accepted | +| What does a blueprint update touch? | [ADR-003](./decisions/003-managed-fields.md) | Accepted | +| Who interprets discovered vertices? | [ADR-004](./decisions/004-processors-propose.md) | Accepted | +| Who may call Inventory and Inspect? | [ADR-005](./decisions/005-app-gateways.md) | Accepted | +| How do blueprint variants combine? | [ADR-006](./decisions/006-variant-merge.md) | Accepted | diff --git a/docs/architecture/blueprints/concepts.md b/docs/architecture/blueprints/concepts.md new file mode 100644 index 0000000..fcf5339 --- /dev/null +++ b/docs/architecture/blueprints/concepts.md @@ -0,0 +1,216 @@ +# Blueprints — Concepts + +Design record for the shipped blueprint package. Field-level usage is in +[reference.md](./reference.md) and [processors.md](./processors.md). + +## 1. What the feature is + +Inventory onboards a device. Inspect places it in the topology and edits +vertices. A blueprint describes how one *kind* of device should be configured +in both places, so a caller can apply that description to many real devices. + +The caller supplies the instance facts (`BlueprintDevice`): source-system id, +label, addresses, credentials, an optional Inventory id, and an optional +Inspect target. The blueprint supplies the type: driver settings, topology +appearance, vertex interpretation, and naming. The engine does not know NetBox +or any other source system. Translation into `BlueprintDevice` stays outside +the package. + +`plan()` reads current server state and returns a reviewable `BlueprintPlan`. +`plan.apply()` writes. The two calls share one executor. `engine.apply(...)` +is `plan` followed by `apply`. + +## 2. Dependency direction + +Document models, naming, and processors do not call VideoIPath. The only +modules that touch `app.inventory` and `app.inspect` are the two gateways +([ADR-005](./decisions/005-app-gateways.md)). `VideoIPathApp` does not depend +on this package at all. + +```mermaid +flowchart LR + caller[Caller] + engine[engine.py] + resolution[resolution.py] + naming[naming.py] + processors[processors/] + inventory[inventory.py] + inspectgw[inspect.py] + apps[InventoryApp and InspectApp] + + caller --> engine + engine --> resolution + engine --> naming + engine --> processors + engine --> inventory + engine --> inspectgw + resolution --> processors + inventory --> apps + inspectgw --> apps +``` + +`BlueprintEngine` is constructed with a `BlueprintApp` protocol (`inventory` +and `inspect` properties). `VideoIPathApp` satisfies that protocol without +being imported here. Construction does not touch the app or the network. + +PyYAML is a normal package dependency. The parser lives in `loader.py`. +`models.py` imports it for `Blueprint.load` / `from_yaml`, so importing +`Blueprint` imports PyYAML. The parser is a dedicated `SafeLoader` subclass: +one UTF-8 document, mapping root, no duplicate keys, aliases, anchors, merge +keys, or custom tags. Limits are 1 MiB, 32 levels, and 100,000 nodes. + +## 3. Module map + +| Module | Responsibility | +| ------ | -------------- | +| `models.py` | Strict document, device facts, patches, plan and result records. | +| `loader.py` | YAML parsing and source locations for errors. | +| `resolution.py` | Variant merge, driver and parameter validation, JSON Schema. | +| `naming.py` | `Text` / `Field` / `Join` expressions and layering. | +| `processors/` | `VertexProcessor`, detached records, per-engine `ProcessorRegistry`, Matrox built-in. | +| `inventory.py` | `InventoryGateway`: one Inventory record, managed-field diff, create/update. | +| `inspect.py` | `InspectGateway`: fresh scoped read, processor run, topology commit, module tags. | +| `engine.py` | Configuration, `plan()`, captured inputs, phased `apply()`. | +| `errors.py` | `BlueprintError` hierarchy. Validation issues carry a path, not raw values. | +| `schemas/blueprint-v1.schema.json` | Generated document schema. `tests/blueprints/test_resolution.py` fails when the file is stale. | + +Public types are re-exported from `blueprints/__init__.py`. Engine internals +(`_Planner`, `_Executor`, `_Captured`, the gateways' work models) are not. + +## 4. From document to writes + +```mermaid +flowchart TD + yaml[Blueprint YAML or dict] + device[BlueprintDevice] + resolved[ResolvedBlueprint] + plan[BlueprintPlan] + apply[apply] + inv[Inventory create or update] + disc[Discovery wait] + sync[Add or synchronize] + topo[Inspect transaction] + tags[Module tag actions] + verify[Read back] + + yaml --> resolved + device --> resolved + resolved --> plan + device --> plan + plan --> apply + apply --> inv --> disc --> sync --> topo --> tags --> verify +``` + +Resolution ([ADR-006](./decisions/006-variant-merge.md)) selects `inventory` and +`topology` independently. A missing section is skipped. The selected named +entry patches `default`, then per-instance `inventory_overrides` patch the +inventory section. Driver `custom_settings` are checked against the driver +model for `SELECTED_SCHEMA_VERSION`. Processor `params` are checked against +the registered `params_model`. The source document is not mutated. Every merged +leaf keeps a provenance string (`default`, `variant:`, `instance`) so a +plan can say where a value came from. + +Naming is layered after that, entry by entry, later sources winning: +library default, engine `naming`, resolved blueprint naming (a topology entry +replaces the same inventory entry), then the `naming` argument on the call. An explicit +`None` turns that entry off. Names are rendered from facts on `BlueprintDevice`, +`module_position`, `EndpointIdentity`, and scalar `attributes`. They are not +read off the current server label, so a second apply cannot grow a second suffix. + +## 5. Planning + +`plan()` deep-copies the device, the blueprint, and the processor registry, and +stores them on the plan together with the resolved document, the layered naming +scheme, and the effective `ApplyOptions`. Later `engine.configure(...)` or +`register_processor(...)` does not change a plan that already exists. + +Planning reads. It does not write, synchronize, stage snapshot edits, or create +catalog tag entries. + +Inventory planning compares managed fields with the current record +([ADR-003](./decisions/003-managed-fields.md)) and records a create, an update, +or no change. A supplied `inventory_id` that does not exist fails. The engine +does not create a replacement record. + +Topology planning needs a target. That is `BlueprintDevice.topology`, or a +`DeviceTarget` built from `inventory_id` when topology was omitted. When the +exact edits cannot be known yet, the topology phases are marked `deferred` and +`fully_resolved` is `False`. That happens when: + +- this plan creates the Inventory record the topology device id will come from +- planned Inventory changes may alter discovery (address, alternate addresses, + credentials, generic settings, custom settings, or `active`) +- the device is not in the topology yet and `sync` allows adding it +- synchronization is pending and this target is allowed to perform it + +A module target does not add or synchronize its parent device. If the parent is +absent, planning fails. If a sync is pending, the plan warns and continues +against the current topology. + +When nothing is deferred, planning reads a detached scope +([ADR-005](./decisions/005-app-gateways.md)), runs the processor +([ADR-004](./decisions/004-processors-propose.md)), and stores the exact edits +plus a fingerprint of the scope. + +The public plan fields (phases, bindings, diagnostics, digest, ids) are the +review surface. `summary()` redacts secrets. `apply()` also uses private +captured work, including unredacted Inventory values, so a plan is an in-memory +object bound to the app it was built with. It is not a document you save and +replay in another process. + +## 6. Applying + +[ADR-002](./decisions/002-plan-then-apply.md) fixes the phase order: + +1. `inventory` — create or update the one record. A new id is stored on the result immediately. +2. `discovery` — poll until the scoped topology can be interpreted, bounded by `discovery_timeout` (default 30 seconds, poll every 1 second). +3. `topology_sync` — add the device or synchronize it, according to `ApplyOptions.sync`. +4. `topology` — device and vertex edits in one `InspectTransaction`, then commit. +5. `module_tags` — `assignTag` / `unassignTag` actions, one tag at a time. These are not part of the topology commit. +6. `verification` — read back what this apply wrote. + +`sync` defaults to `add_only`. `none` requires the device to already be in the +topology. `add_only` adds the device and synchronizes new elements; if the +pending sync would update or remove elements, apply fails instead of escalating. +`reconcile` allows a full sync. The gateway calls `sync_devices` without a +conflict strategy, so Inspect's default `ConflictStrategy.STRICT` applies. +A module target never reaches this add/sync path. + +Unchanged phases are skipped. An apply that would change nothing writes nothing. + +`dry_run=True` runs the same stale-plan, conflict, and pending-edit checks, then +returns before any write. Deferred topology stays `deferred` on a dry run, +because those edits depend on writes the dry run does not perform. `status` is +`planned` when writes would have run, and `no_change` otherwise. + +Before an Inventory update, the gateway re-reads the record and compares +fingerprints of the managed fields. Before a topology commit that was fully +resolved at plan time, it re-reads the scope and compares the fingerprint, +including sibling endpoint labels that the collision check depends on. +Module tags are re-read immediately before their phase. Pending uncommitted +Inspect edits that overlap the plan's device, vertices, or module are rejected. +The Inspect transaction then does its own baseline check. These checks are +client-side. A short gap between the read and the write remains. + +If a later phase fails, earlier phases stay applied. `BlueprintApplyError.result` +is `partial` when something was written, `failed` when nothing was, and +`unknown` when a write raised and the client cannot prove the server rejected +it (`InspectCommitError`, `InspectCommitConflictError`, `BlueprintError`, and +known Inventory `ValueError` messages count as known rejections, including a +failed re-read that happens before `update_device` sends the update). A failed +synchronization lookup is also an error; it is not treated as "nothing pending". +Verification does not roll a write back. A mismatched or failed read-back, or a +later phase that fails before the topology read-back runs, sets `verification` +to `unconfirmed` and leaves the writes in place. + +## 7. What stays outside + +The package does not select a processor from the driver id, prune configuration +it used to manage, roll back a partial apply, migrate drivers, infer cables, or +provision services. Catalog tags are referenced by id and are not created. +Endpoint label uniqueness is checked per device, including endpoints outside +the changed set, and not across the system. `ApplyOptions(naming_collisions="allow")` +is the escape hatch. Names are not truncated or given a numeric suffix. + +Offline tests under `tests/blueprints/` drive the engine with fakes. Live +behavior of the writes still has to be checked on a test system. diff --git a/docs/architecture/blueprints/decisions/001-standalone-feature.md b/docs/architecture/blueprints/decisions/001-standalone-feature.md new file mode 100644 index 0000000..d599bc2 --- /dev/null +++ b/docs/architecture/blueprints/decisions/001-standalone-feature.md @@ -0,0 +1,37 @@ +# ADR-001: Standalone feature, not a VideoIPathApp property + +> Status: **Accepted** + +## Context + +Inventory, topology, preferences, profile, and security are properties of +`VideoIPathApp`. Blueprints sit on top of Inventory and Inspect and are optional +for callers who only onboard devices by hand. + +## Options + +- Add `app.blueprints`, initialized lazily like the other apps. +- Keep a separate `BlueprintEngine` that receives an app. + +## Decision + +The feature is a separate package object. `VideoIPathApp` neither imports nor +exposes it. Callers write `BlueprintEngine(app)`. + +The engine depends on a `BlueprintApp` protocol with `inventory` and `inspect` +properties. Any object with those properties can be passed in. Construction +does not read the app or open a connection. + +Processor registrations, `ApplyOptions`, and naming defaults live on the engine +instance. Two engines may share one app and still keep separate registries. +There is no process-global processor map. + +## Consequences + +- The rest of the package can be imported and released without pulling blueprint + call sites into `VideoIPathApp`. +- Tests can pass a small fake that satisfies the protocol. +- Discoverability is worse than `app.blueprints`. The getting-started page and + this record are the entry points. +- Sharing an app across engines does not serialize writes. Callers who apply + concurrently against one server have to coordinate themselves. diff --git a/docs/architecture/blueprints/decisions/002-plan-then-apply.md b/docs/architecture/blueprints/decisions/002-plan-then-apply.md new file mode 100644 index 0000000..711f440 --- /dev/null +++ b/docs/architecture/blueprints/decisions/002-plan-then-apply.md @@ -0,0 +1,54 @@ +# ADR-002: Plan, then apply, without a spanning transaction + +> Status: **Accepted** +> +> Field ownership: [ADR-003](./003-managed-fields.md). Gateways: +> [ADR-005](./005-app-gateways.md). + +## Context + +Callers need to review driver settings, names, and vertex edits before anything +is written. Inventory creates and Inspect commits are different server +operations. Driver discovery after an Inventory write is not available in the +same response. + +## Options + +- Write immediately inside `apply()`, with no stored preview. +- One client-side transaction that rolls every phase back on failure. +- A frozen plan, then a phased apply that reports how far it got. + +## Decision + +`engine.plan(...)` is read-only and returns an immutable `BlueprintPlan`. +`plan.apply()` takes no configuration. It re-checks the captured baselines and +executes that plan. `engine.apply(...)` is plan plus apply. + +Phases run in order: inventory, discovery, topology sync, topology commit, +module tags, verification. Absent or unchanged phases are skipped. Inventory +and Inspect writes are separate. Nothing rolls back an earlier phase. + +When topology edits depend on earlier work, the plan marks those phases +`deferred` and sets `fully_resolved` to `False`. During apply the engine waits +for discovery, then computes the edits from the blueprint, parameters, naming, +and target captured on the plan. A dry run does not invent those edits. It +reports the phases as `deferred`. + +`dry_run=True` runs the stale-plan, conflict, and pending-edit checks and +performs no write. + +## Consequences + +- A reviewed plan cannot be silently retargeted. Changing options or variants + means a new plan. +- A plan holds private captured state, including credentials, and the executor + bound to the app. It is not a serializable job description. +- A failure after Inventory create is `partial`. The result carries + `inventory_id` so the caller can bind it and retry without creating a second + record. +- A write that raises, when the client cannot prove the server rejected it, is + `unknown`. The caller reads the server back (a new plan does that) before + retrying. +- There is no automatic undo. Callers that need a single all-or-nothing change + have to split scopes themselves: apply inventory, add the device, then review + a fully resolved topology plan. diff --git a/docs/architecture/blueprints/decisions/003-managed-fields.md b/docs/architecture/blueprints/decisions/003-managed-fields.md new file mode 100644 index 0000000..f2acbf2 --- /dev/null +++ b/docs/architecture/blueprints/decisions/003-managed-fields.md @@ -0,0 +1,51 @@ +# ADR-003: Updates touch only managed fields + +> Status: **Accepted** + +## Context + +A blueprint is reused across devices and edited over time. Callers also set +fields in the VideoIPath UI, or with other scripts, that the blueprint does not +mention. An update has to change the fields the blueprint owns without wiping +the rest. + +## Options + +- Treat the blueprint as a full desired document and reset every omitted field. +- Remember the last applied document and revert fields that disappeared from it. +- Write only fields the resolved blueprint, the instance facts, or naming set + on this run. + +## Decision + +Absence means unmanaged. An update deep-copies the current Inventory record and +overlays managed fields only. The same rule applies to topology patches: +`DevicePatch`, `ModulePatch`, and `VertexPatch` reject `null`, because `null` +would be ambiguous with "leave it alone". Explicit `false`, `0`, `[]`, and +`""` are values. `[]` clears a list that is managed, such as local tags or +alternate addresses. + +There is no stored last-applied state. Removing a key from a blueprint, or +switching variant, does not put the previous value back. A variant that must +turn something off has to set the new value. + +Naming follows the same rule. An entry left unset is unmanaged. An entry set +to `None` in a later naming layer turns that name off. Rendered names come from +caller facts and endpoint identity, not from the label currently on the server. + +The Inventory round-trip preserves what `InventoryDevice` represents, which is +the same boundary as `InventoryApp.update_device`. Wire fields those models do +not parse are dropped. Once a managed change exists, the engine calls +`update_device(..., compare_config=False)` so an explicit credential change is +not swallowed by Inventory's own authentication filter. + +## Consequences + +- A second apply of an unchanged blueprint is a no-op for fields that already + match, and it does not rename by appending to the current label. +- The engine cannot garbage-collect a tag or a setting it applied last year + and no longer mentions. Pruning is a caller decision: set the new value. +- Plans can show provenance per managed field. They cannot show "this server + value came from a previous blueprint". +- Secrets are redacted in `FieldChange` and in `summary()`, and they are kept + on the private captured work used to write. diff --git a/docs/architecture/blueprints/decisions/004-processors-propose.md b/docs/architecture/blueprints/decisions/004-processors-propose.md new file mode 100644 index 0000000..cfa6a37 --- /dev/null +++ b/docs/architecture/blueprints/decisions/004-processors-propose.md @@ -0,0 +1,58 @@ +# ADR-004: Processors propose, the engine writes + +> Status: **Accepted** +> +> Where the proposal is computed: [ADR-005](./005-app-gateways.md). + +## Context + +Device types do not share a vertex layout. Matrox ConvertIP endpoints are not +the same shape as the next driver. That interpretation changes more often than +the write path, and it has to be replaceable per engine without editing +Inventory or Inspect. + +## Options + +- Teach the engine one layout per `driver_id`. +- Let a processor call `app.inspect` and write the vertices it wants. +- Let a processor return a typed proposal. The engine validates, names, + compares, and writes. + +## Decision + +A `VertexProcessor` implements `process(context, params) -> ProcessorResult`. +`params` are a strict Pydantic model declared as `params_model`. A new instance +is created for each run. + +`ProcessingContext` is a frozen snapshot of one device or one module: source +facts without credentials, ports, vertices, resolved interface bindings, and +optional driver context. It is not a reference to the live Inspect snapshot. A +module target's context contains only that module's vertices. Returning an id +outside the scope fails. + +The engine rejects device patches on a module target, module patches on a +device target, conflicting proposals (identical duplicates are merged), and +kind-specific fields on the wrong vertex kind (`sdp_support` and destination +ports need a codec vertex, `supports_static_igmp` an IP vertex). Explicit `topology.device` values override +processor device defaults. Naming overrides proposed labels. Explicit `vertices` +overrides win over both. + +`ProcessorInputError` means the layout is unsupported or not ready yet. While +topology work is deferred, the engine retries that error until +`discovery_timeout`. + +Registration is per engine ([ADR-001](./001-standalone-feature.md)). Duplicate +ids fail, including the built-in `matrox.convertip.default`. YAML names the +registered id. It does not name a Python import path. The built-in is included +unless a registry is built with `include_builtins=False`. + +## Consequences + +- New device types are new classes, not new branches in the executor. +- Processors are trusted code in the caller's process. The contract stops + accidental writes through the engine. It does not sandbox a processor that + opens its own network connection. +- The engine does not pick a processor by driver id. The blueprint names + `processor_type`. An Inventory-only plan does not need a topology processor. +- `engine.validate()` resolves every variant and reports processors that are + not registered. Planning only requires the processor for the selected scope. diff --git a/docs/architecture/blueprints/decisions/005-app-gateways.md b/docs/architecture/blueprints/decisions/005-app-gateways.md new file mode 100644 index 0000000..bf3e1ac --- /dev/null +++ b/docs/architecture/blueprints/decisions/005-app-gateways.md @@ -0,0 +1,55 @@ +# ADR-005: Inventory and Inspect are reached through two gateways + +> Status: **Accepted** +> +> Why the feature is not on the app: [ADR-001](./001-standalone-feature.md). + +## Context + +The engine needs current Inventory records, a scoped Inspect topology, an +Inspect commit, and module tag actions. Inspect also keeps a live snapshot with +uncommitted domain-object edits. Reading that snapshot during planning would +treat a caller's unsaved edits as server truth, or flush them. + +## Options + +- Call `app.inventory` and `app.inspect` from the planner and the executor + wherever a read or write is needed. +- Confine every app call to `inventory.py` and `inspect.py`. + +## Decision + +`InventoryGateway` is the only object that calls `app.inventory`. It reads one +record, resolves an SNMP configuration by exact label or id, creates with +`add_device`, and updates with `update_device`. Label lookups use +`find_device_id_by_label`. Address conflict checks use +`find_device_ids_by_addresses`, one read for every candidate spelling. An empty +inventory response is no match. + +`InspectGateway` is the only object that calls `app.inspect`. Scope data is +built from fresh collector and lookup reads (`get_device_detail`, +`lookup_inspect_device`, `lookup_vertices`), not from the live snapshot's +domain objects. Pending edits are visible only through `staged_edit_keys()`, +which is used to reject overlap, not to seed the plan. + +Topology writes go through one `InspectTransaction`: `update_device` and +`update_vertex` intents, then `commit()`. Module tags use `assignTag` and +`unassignTag` on the internal Inspect API handle, because those actions are +not part of `updateTopology`. After tag writes, the gateway asks the snapshot, +if one is loaded, to refresh that device. + +Processors never receive a gateway. They receive `ProcessingContext` +([ADR-004](./004-processors-propose.md)). + +## Consequences + +- A test can replace the two app surfaces without standing up HTTP. +- A change to Inventory or Inspect call style has two files to update, not the + planner. +- `InspectGateway` reaches `_inspect_api` for detail reads and tag actions. + That is a deliberate crack in the public-method rule, limited to this + module, because the public Inspect app does not expose those calls in the + shape the engine needs. +- Fresh reads can disagree with the snapshot the caller is looking at. That + is intentional. The plan describes server state. Overlapping uncommitted + edits fail the apply until the caller commits or discards them. diff --git a/docs/architecture/blueprints/decisions/006-variant-merge.md b/docs/architecture/blueprints/decisions/006-variant-merge.md new file mode 100644 index 0000000..0978572 --- /dev/null +++ b/docs/architecture/blueprints/decisions/006-variant-merge.md @@ -0,0 +1,58 @@ +# ADR-006: Variants are a fixed patch over `default` + +> Status: **Accepted** +> +> What a merged field means at write time: [ADR-003](./003-managed-fields.md). + +## Context + +One device type often has a handful of layouts (a receiver against a sender, +for example) that share most of their driver settings. Authors will reach for +YAML anchors, merge keys, and `extends` unless the document says how composition +works. + +## Options + +- General composition: anchors, merge keys, includes, and list concatenation. +- One rule: `default`, then a named patch, then instance overrides. No other + composition. + +## Decision + +`inventory_variant` and `topology_variant` are selected independently. Both +default to `default`. An unknown name fails and the error lists the names that +exist. There is no fallback. + +A named entry patches that section's `default`: + +- mappings merge by key +- scalars replace +- lists replace as a whole +- naming expressions and tag specifications replace as a whole +- `vertex_processor: null` disables an inherited processor +- a different `processor_type` replaces the whole processor declaration, + parameters included +- a different `driver_id` drops the inherited `custom_settings` + +`inventory_overrides` on `BlueprintDevice` are a third patch, applied only to +the inventory section, and recorded as provenance `instance`. + +The loader rejects YAML aliases, anchors, and merge keys, so a document cannot +smuggle a second composition language past the merge rules. Documents that +need generation are built in Python and passed to `Blueprint.from_dict`. + +`schema_version` is part of the document model. The published JSON Schema is +`schemas/blueprint-v1.schema.json`, regenerated from `Blueprint.json_schema`. +The unit test fails when the committed file drifts. + +## Consequences + +- A variant file is readable without resolving anchors. What you see in the + named entry is the patch, not a full copy. +- Authors cannot share a list fragment across variants. They repeat the list, + or they generate the document in Python. +- Changing merge behavior later would change the meaning of existing files. + That is why this is a decision record rather than a loader detail. +- `engine.json_schema()` adds the parameter schemas of processors registered + on that engine and limits `processor_type` to them. The packaged file does + not include third-party processors and accepts any `processor_type`. diff --git a/docs/architecture/blueprints/decisions/README.md b/docs/architecture/blueprints/decisions/README.md new file mode 100644 index 0000000..71d91de --- /dev/null +++ b/docs/architecture/blueprints/decisions/README.md @@ -0,0 +1,16 @@ +# Architecture Decision Records — Blueprints + +Each ADR records one decision that would be expensive to reverse: the context, +the choice, and the consequences. Supersede a decision with a new ADR rather +than rewriting history. + +## Index + +| ADR | Title | Status | +| --- | ----- | ------ | +| [001](./001-standalone-feature.md) | Standalone feature, not a VideoIPathApp property | Accepted | +| [002](./002-plan-then-apply.md) | Plan, then apply, without a spanning transaction | Accepted | +| [003](./003-managed-fields.md) | Updates touch only managed fields | Accepted | +| [004](./004-processors-propose.md) | Processors propose, the engine writes | Accepted | +| [005](./005-app-gateways.md) | Inventory and Inspect are reached through two gateways | Accepted | +| [006](./006-variant-merge.md) | Variants are a fixed patch over `default` | Accepted | diff --git a/docs/architecture/blueprints/processors.md b/docs/architecture/blueprints/processors.md new file mode 100644 index 0000000..e3ece3e --- /dev/null +++ b/docs/architecture/blueprints/processors.md @@ -0,0 +1,152 @@ +# Blueprints — Vertex processors + +A processor reads the vertices one driver discovered and proposes configuration. +The engine validates the proposal, applies naming, compares it with the server, +and writes ([ADR-004](./decisions/004-processors-propose.md)). The rest of the +blueprint reference is in [reference.md](./reference.md). + +## 1. Writing a processor + +```python +from pydantic import BaseModel, ConfigDict + +from videoipath_automation_tool.blueprints import ( + EndpointIdentity, ProcessingContext, ProcessorResult, VertexEdit, VertexPatch, VertexProcessor, +) + + +class SimpleParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + factory_label: str + + +class SimpleVideoProcessor(VertexProcessor[SimpleParams]): + params_model = SimpleParams + + def process(self, context: ProcessingContext, params: SimpleParams) -> ProcessorResult: + vertex = context.require_vertex(kind="codec", factory_label=params.factory_label, vertex_type="In") + return ProcessorResult( + vertices=[ + VertexEdit( + vertex_id=vertex.id, + fields=VertexPatch(use_as_endpoint=True), + endpoint=EndpointIdentity(direction="TX", media="video", index=1), + ) + ] + ) + + +engine.register_processor("example-org.simple-video", SimpleVideoProcessor) +``` + +```yaml +topology: + default: + vertex_processor: + processor_type: example-org.simple-video + params: {factory_label: Video Sender} +``` + +A processor subclasses `VertexProcessor[ParamsT]`, declares `params_model` (a +Pydantic model class), and implements `process(context, params)`. The engine +validates the YAML `params` against `params_model` before calling `process`, and +creates a new processor instance for every run, so an instance may keep +run-local state. + +## 2. Input: `ProcessingContext` + +`ProcessingContext` is a detached, immutable snapshot of the target scope. + +| Field | Content | +|---|---| +| `source` | `SourceFacts`: `key`, `label`, `description`, `module_position`, `attributes` from `BlueprintDevice`. No credentials. | +| `scope`, `device_id`, `module_id` | `"device"` or `"module"`, and the target ids. | +| `device` | `DeviceRecord`: label, description, factory label, icon, SDP strategy, site, coordinates, tags. | +| `modules`, `ports`, `vertices` | `ModuleRecord`, `PortRecord`, `VertexRecord` tuples, limited to the target scope. A module target sees only its own module. | +| `interfaces` | The resolved `ip_vertex_mapping` as `InterfaceBinding`s (`key`, `candidate`, `port_id`, `in_vertex_id`, `out_vertex_id`). | +| `owner` | `DriverContext` of the topology owner's Inventory record: `inventory_id`, `driver_id`, `address`, `custom_settings`. | +| `inventory` | The source entity's own `DriverContext` when it differs from the owner (a module with its own record). | + +`VertexRecord` carries `id`, `port_id`, `module_id`, `factory_label`, +`port_label`, `vertex_type` (`In`, `Out`, `Internal`, `Undecided`), `kind` +(`codec`, `ip`, `router`, `generic`), +the current `label`, `description`, `use_as_endpoint`, `active`, `sips_mode`, +`tags`, `codec_format`, `media_type`, and the kind-specific fields. + +Helpers: + +| Method | Returns | +|---|---| +| `vertex(vertex_id)` | The in-scope vertex, or `None`. | +| `find_vertices(*, kind=None, vertex_type=None, factory_label=None, module_id=None)` | Every in-scope match (exact comparisons), ordered by id. | +| `require_vertex(...)` | The single match. Raises `ProcessorInputError` for zero or several. | + +## 3. Output: `ProcessorResult` + +Return `ProcessorResult(vertices=[VertexEdit, ...], device=DevicePatch | None, module=ModulePatch | None, diagnostics=[...])`. + +- `VertexEdit(vertex_id, fields=VertexPatch(...), endpoint=EndpointIdentity(...))`. `VertexPatch` uses the same fields as YAML vertex overrides ([reference §3](./reference.md#vertex-fields)). +- Give each endpoint an `EndpointIdentity(direction, media, index, engine=None, program=None, leg=None)` and let naming build the labels. +- `DevicePatch` is valid on device targets only, `ModulePatch` on module targets only. +- `Diagnostic(level="info" | "warning", code, message, entity_id=None)` shows up on the plan and the result. + +The engine rejects ids outside the target scope, device patches on a module +target, and conflicting proposals. Identical duplicate proposals are merged. +Kind-specific fields on the wrong vertex kind (for example `sdp_support` on an IP +vertex) are rejected. + +Precedence: explicit `topology.device` values override processor defaults, +naming overrides proposed labels, and explicit `vertices` overrides win over +both. + +Raise `ProcessorInputError` when the layout is unsupported or still incomplete. +While discovery is deferred, the engine retries until `discovery_timeout`. + +## 4. Registration + +Registration belongs to one engine (`BlueprintEngine(app, processors={...})` or +`engine.register_processor(id, cls)`). Ids use letters, digits, `.`, `_`, and +`-`. Registering an id that already exists, including a built-in id, fails. YAML +names the registered id; it does not name a Python import path. + +A processor is trusted code running in your process. It is not sandboxed. Keep +it free of network calls and source-system imports. + +`ProcessorRegistry` is the standalone registry type. It includes the built-ins +by default (`include_builtins=True`) and is what `Blueprint.validate_full(registry)` +and `Blueprint.json_schema(registry, restrict_processor_types=...)` take. `engine.registry` returns a copy of +the engine's registrations. + +## 5. Porting a `TopologyDevice`-based processor + +- Report problems as returned `Diagnostic` values, or with ordinary logging, rather than through the script-manager logger. +- Read vertices with `context.find_vertices(...)`. The context is a snapshot, so there is no mutable `TopologyDevice` to walk. +- Describe each endpoint with `EndpointIdentity`, and pass module context through `module_position` and `attributes`. Put the label convention in a `NamingScheme` instead of parsing a base label or splitting on `-MODULE-`. +- Propose `VertexEdit` and `DevicePatch` values. The processor should not mutate vertex attributes itself. +- Replace `kwargs` with a strict Pydantic `params_model`, and write the previous defaults into that model. An omitted tag parameter leaves tags unmanaged. Older processors often cleared them when the argument was missing, so set `[]` when clearing is still what you want. +- Keep the processor free of network calls and source-system imports. + +## 6. Built-in: `matrox.convertip.default` + +Class `MatroxConvertIPProcessor`, parameters `MatroxConvertIPParams`. + +The processor recognizes three layouts. **tx** is one video codec vertex and one +audio codec vertex, both `In`. **rx** is the same pair, both `Out`. +**four_split** is four `Out` video receivers whose factory labels are +`Video Receiver 0` through `Video Receiver 3` (indices 1–4), plus one audio +receiver. Media is taken from the codec format, then from `Video` or `Audio` in +the factory label. Any other mix fails. The processor does not skip vertices it +does not recognize. + +| Parameter | Default | Meaning | +|---|---|---| +| `mode` | `auto` | `tx`, `rx`, or `four_split` checks that the discovered layout matches. The parameter does not switch the hardware mode. | +| `video_receiver_tags`, `video_sender_tags`, `audio_receiver_tags`, `audio_sender_tags` | omitted | Category tags. Leave the parameter out to leave tags alone, pass `[]` to clear them, or pass `{add, remove}` to add and remove. | +| `redundant_streams` | omitted | `true` sets `SIPSAuto`, `false` sets `NONE`, and omitting the parameter leaves the value alone. `true` requires exactly two streaming IP vertices. Use `ip_vertex_mapping` when the device also has management ports. The processor does not check that paths actually run through that pair. | +| `allow_null_audio_label` | `false` | When `true`, the single codec vertex labelled exactly `null`, which cannot be classified, is accepted as audio. The processor emits a warning diagnostic. | +| `configure_control` | `false` | Not available yet. Setting it raises `BlueprintCapabilityError`. The Inspect control mapping is still unverified, so control settings on the device stay as they are. | + +The proposal enables the endpoints and assigns an `EndpointIdentity` with engine +`0` and program `1`. The factory label becomes the description, joined with the +device description when you supplied one. Senders get SDP support. On a device +target the icon is `gateway` for tx, and `monitor` for rx and four-split. diff --git a/docs/architecture/blueprints/reference.md b/docs/architecture/blueprints/reference.md new file mode 100644 index 0000000..c0e70ae --- /dev/null +++ b/docs/architecture/blueprints/reference.md @@ -0,0 +1,583 @@ +# Blueprints — Reference + +Complete user reference for `videoipath_automation_tool.blueprints`. For a short +introduction, read the [getting-started page](../../getting-started-guide/05_Blueprints.md) +first. Why the feature works this way is in [concepts.md](./concepts.md) and the +[decisions](./decisions/README.md). Vertex processors, including the built-in +Matrox processor, have their own page: [processors.md](./processors.md). + +## Contents + +1. [Engine API](#1-engine-api) +2. [Describing a device: `BlueprintDevice`](#2-describing-a-device-blueprintdevice) +3. [Blueprint document](#3-blueprint-document) +4. [Variants](#4-variants) +5. [Loading and validation](#5-loading-and-validation) +6. [Naming](#6-naming) +7. [Planning and applying](#7-planning-and-applying) +8. [Results, errors, and recovery](#8-results-errors-and-recovery) +9. [Engine configuration](#9-engine-configuration) +10. [Source-system adapters](#10-source-system-adapters) +11. [Limits and verification status](#11-limits-and-verification-status) +12. [Public API index](#12-public-api-index) + +## 1. Engine API + +`BlueprintEngine(app, *, processors=None, options=None, naming=None, logger=None)` +takes any object with `inventory` and `inspect` properties (the `BlueprintApp` +protocol). `VideoIPathApp` satisfies it. The app has no blueprint attribute, so +you construct the engine yourself. + +| Member | Purpose | +|---|---| +| `plan(device, blueprint, *, scope="all", inventory_variant="default", topology_variant="default", naming=None, options=None)` | Read-only. Loads and resolves the blueprint, reads current state, returns a `BlueprintPlan`. | +| `apply(device, blueprint, *, ..., dry_run=False)` | Same arguments as `plan`, plus `dry_run`. Identical to `plan(...).apply(dry_run=...)`. | +| `validate(blueprint)` | Offline. Resolves every variant and reports unregistered processors. | +| `json_schema()` | Document JSON Schema for this engine: `processor_type` limited to the registered processors, with each one's parameters. | +| `register_processor(processor_id, processor_cls)` | Registers a processor for this engine only. See [processors.md](./processors.md). | +| `configure(*, options=..., naming=...)` | Replaces engine defaults. See [section 9](#9-engine-configuration). | +| `options`, `naming`, `registry`, `app` | Current defaults. `registry` returns a copy. | + +`scope` is `"all"`, `"inventory"`, or `"topology"`. The blueprint argument accepts +a file path string, a `pathlib.Path` (or another `os.PathLike[str]`), or a +`Blueprint` instance. + +**`BlueprintPlan`** + +| Member | Meaning | +|---|---| +| `apply(*, dry_run=False)` | Re-checks the plan's assumptions and executes it. A dry run performs every check but no write. | +| `summary()` | Human-readable, redacted before/after listing, bindings, diagnostics, and unresolved work. | +| `phases`, `phase(name)` | `PlannedPhase` records (`planned`, `no_change`, `deferred`, `skipped`) with their `PlannedOperation`s and `FieldChange`s. | +| `has_changes` | `True` when any phase is `planned` or `deferred`. | +| `fully_resolved` | `False` when topology work is deferred until apply. | +| `interface_bindings`, `diagnostics`, `skipped_sections` | Resolved `ip_vertex_mapping`, warnings, and sections the scope skipped. | +| `source_key`, `scope`, `inventory_variant`, `topology_variant`, `inventory_id`, `topology_target`, `driver_id`, `driver_schema_version`, `processor_type`, `blueprint_digest` | Identity of what was planned. | + +A plan is an in-memory object bound to the app it was built with. It also holds +unredacted Inventory values for the write, so it is not a document you save and +replay elsewhere. + +**`ApplyResult`** + +| Field | Meaning | +|---|---| +| `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. | +| `ok` | `True` for `succeeded`, `no_change`, and `planned`. | +| `inventory_id`, `topology_device_id`, `module_id`, `source_key` | Ids involved. Persist `inventory_id` in your source system. | +| `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. | +| `interface_bindings`, `diagnostics` | As on the plan, after deferred work was materialized. | +| `materialized` | `True` when deferred topology work was computed during apply. | +| `verification`, `verification_detail` | `confirmed`, `unconfirmed`, or `not_applicable`. | +| `dry_run` | Whether this was a dry run. | + +## 2. Describing a device: `BlueprintDevice` + +`BlueprintDevice` is one device, module, or virtual device, plus the facts only +your source system knows. + +| Field | Meaning | +|---|---| +| `key` | Your id for this device, for example the id in the source system, so you can match a result back to it. It is not written to VideoIPath. | +| `label`, `description` | Instance facts used by naming. | +| `inventory_id` | The Inventory record that already belongs to this device. If that record is missing, planning fails. The engine will not create a second one in its place. Leave the field out when a selected inventory section should create the record. | +| `management_address`, `alternative_addresses` | Connection addresses. The driver decides the string format, so these are not always IP addresses. An alternative address is a string or an `AlternativeAddress(address, credentials)`. Leave the field out to leave the server value alone. Pass `[]` to clear alternative addresses. A supplied list is the full desired set and also replaces per-address credentials: a plain string carries none, so an existing credential is cleared unless that address is an `AlternativeAddress`. An address listed twice fails. | +| `credentials` | Runtime `Credentials(username, password)`. Credentials stay out of YAML, and the engine redacts them in plans, results, and errors. | +| `topology` | `DeviceTarget(device_id)` or `ModuleTarget(device_id, module_id)`, using exact Inspect ids. Leave it out to target the Inventory id, including an id this same plan creates. | +| `module_position` | Module context you supply for naming and mappings. It is a string, such as `"0"`, `"A1"`, or `"1/2"`. | +| `inventory_overrides` | Per-device Inventory settings that are not secrets (`InventorySettings`): `custom_settings`, `generic_settings`, `snmp`, `metadata`, `active`. Applied after the selected variant. | +| `attributes` | JSON facts you own. Scalar leaves are available to naming as `attributes.`, and processors see them on `context.source`. | + +`BlueprintDevice.from_inventory(inventory_device, key=None)` binds an existing +Inventory device by id, label, and description. + +These are the arrangements the engine supports: + +- **Standalone device.** Pass `inventory_id`, or let the inventory section create the record. Topology defaults to that device. +- **Chassis.** Use a device-scoped blueprint for the chassis. Call the engine again for each module, with a `ModuleTarget` and that module's own blueprint. +- **Module with no Inventory record.** Pass a `ModuleTarget` and a topology-only blueprint, or set `scope="topology"`. +- **Module with its own Inventory record.** `inventory_id` names the module's record. `ModuleTarget` names the module inside its parent. The two ids do not have to match. The parent device stays as it is. +- **Several topology scopes for one source entity.** Call the engine more than once with the same `key` and different targets. Configure Inventory on one of those calls. +- **Existing virtual device.** Pass `DeviceTarget("virtual.N")` and `scope="topology"`. + +Take module ids from Inspect (`device.modules`). A module label or a slot label is +a display name. The engine does not use it as identity. + +## 3. Blueprint document + +```yaml +schema_version: 1 + +inventory: + default: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + generic_settings: {enable_https: false, http_auth: 0} + custom_settings: {port: 8080, disable_rx_sdp_with_null: true} + +topology: + default: + device: {icon_size: medium} + ip_vertex_mapping: + stream-a: [P1] + stream-b: [P2] + vertex_processor: + processor_type: matrox.convertip.default + params: + redundant_streams: true + video_receiver_tags: [Format~~video-tag-a] + video_sender_tags: [Format~~video-tag-b] + receiver: + vertex_processor: + params: {mode: rx} +``` + +`schema_version` must be `1`. `inventory` and `topology` are each optional, and +the file needs at least one of them. A section you include must contain a +`default` entry. Variant names start with a letter or digit and may contain +letters, digits, `_`, `.`, and `-`. + +### Inventory entry + +| Key | Effect | +|---|---| +| `driver_id` | Required on the resolved entry. Checked against the driver schema selected in this library. It belongs on the entry, not inside `custom_settings`. | +| `custom_settings` | Driver settings, keyed by the Python field name (`port`, for example). An unknown name, a wrong type (`"80"` where an integer is required), or a value outside the allowed range fails the plan before any write. | +| `generic_settings` | `enable_https`, `trust_all_certificates`, and `http_auth`. `http_auth` is the server's raw `httpAuth` mode code. `0` turns HTTP authentication off. | +| `snmp` | `use_global_settings`, and `configuration` as an exact label (a string) or `{id: ...}`, for example `{id: default}`. A missing or ambiguous label fails. | +| `active` | Enables or disables the record. | +| `metadata` | Patches `meta` by key. Values are strings, numbers, or booleans. Keys you leave out stay as they are. | +| `naming` | `inventory_label` and `inventory_description`. See [section 6](#6-naming). | + +### Topology entry + +| Key | Effect | +|---|---| +| `device` | Device targets only: `icon_type`, `icon_size`, `sdp_strategy`, `site_id`, `coordinates` with `x` and `y`, and `tags`. | +| `module` | Module targets only: `tags` on that module. | +| `ip_vertex_mapping` | Your interface key, mapped to an ordered list of candidate port factory labels. The first candidate that matches exactly one IP port in scope wins. A candidate that matches more than one port fails the plan, and later candidates are not tried. Labels may contain `{module.position}`. Bindings show up in the plan and the result for cable workflows. The mapping itself writes nothing. | +| `vertex_processor` | `processor_type` and `params`. The registered processor validates `params`. See [processors.md](./processors.md). | +| `vertices` | Explicit overrides. Point at the vertex with `vertex_id`, or with `factory_label` (may contain `{module.position}`) optionally narrowed by `kind` and `direction` (`In`, `Out`, `Internal`, `Undecided`), and supply `fields`. Those fields use the same contract as processor output. Each override must match exactly one vertex. | +| `naming` | `device_label`, `device_description`, `endpoint_label`, `endpoint_description`. | + +### Vertex fields + +On a vertex override or in processor output, `fields` may set: + +| Field | Applies to | +|---|---| +| `use_as_endpoint`, `label`, `description`, `active`, `sips_mode`, `tags` | All vertices. `""` clears a label or a description. | +| `sdp_support`, `main_destination_port`, `spare_destination_port` | Codec vertices. Ports are `0`–`65535`. | +| `supports_static_igmp` | IP vertices. | + +Explicit `null` is rejected on any patch field. Omit the field instead. + +### Tags + +Tags are exact catalog ids, such as `Format~~V_1080p50`. The engine does not +create catalog entries. + +- A list replaces the local tags exactly, and `[]` clears them. +- `{add: [...], remove: [...]}` changes the listed tags and leaves every other local tag in place. A tag may not appear in both lists, and at least one list must be non-empty. +- Inherited tags stay as they are. Only local tags are edited. + +### Leaving a field out + +**Leaving a field out leaves the server value alone.** An update writes only the +fields the resolved blueprint, the instance facts, or naming actually set. +Everything else on the record stays as it is. Delete a field from the blueprint, +or switch variants, and values written by an earlier configuration remain on the +server. The engine does not store what it applied last time. If a variant should +turn a stream off, replace a tag set, or clear an endpoint flag, put that new +value in the variant. Background: [ADR-003](./decisions/003-managed-fields.md). + +## 4. Variants + +`inventory_variant` and `topology_variant` are chosen independently. Both default +to `default`. An unknown name fails, and the error lists the names that exist. +There is no silent fallback to `default`. + +A named entry is a patch on top of that section's `default` +([ADR-006](./decisions/006-variant-merge.md)): + +- Mappings merge by key. A scalar replaces the inherited value. A list replaces the inherited list as a whole, so the two lists are not concatenated. +- A naming expression replaces the inherited expression. A tag specification does the same. +- `vertex_processor: null` turns off a processor inherited from `default`. A different `processor_type` replaces the whole processor declaration, parameters included. +- A different `driver_id` drops the inherited `custom_settings`. +- `false`, `0`, `[]`, and `""` are values you set on purpose. Leaving the key out is a different choice. + +Per-device `inventory_overrides` are applied after the variant. + +The loader rejects `extends`, imports, YAML anchors, and merge keys. Each document +is complete by itself. When you need to preprocess one, build a dictionary in +Python and pass it to `Blueprint.from_dict`. + +## 5. Loading and validation + +Pass a file directly to any engine entry point: + +```python +from pathlib import Path + +engine.validate("blueprints/matrox-convertip.yml") # optional: validate every variant offline +plan = engine.plan(device, Path("blueprints/matrox-convertip.yml")) +``` + +Strings always mean file paths, and relative paths use the current working +directory. For raw YAML text, pass `Blueprint.from_yaml(text)` explicitly. Use +`Blueprint.load(path)` when you want to load once and reuse a blueprint across +many calls. + +| Constructor | Input | +|---|---| +| `Blueprint.load(path)` | A YAML file. | +| `Blueprint.from_yaml(text, *, source=None)` | YAML text or bytes. `source` names it in error messages. | +| `Blueprint.from_dict(mapping, *, source=None)` | An already-parsed mapping. | + +These check the document structure only. They do not connect to a server. +`blueprint.variant_names("inventory" | "topology")` lists the variants. + +Files are loaded once per engine call. A plan retains the loaded configuration, +so `plan.apply()` does not read the file again even if it has changed or been +deleted. Create a new plan to use updated file contents. Missing files raise the +normal file error; YAML validation errors retain their source locations. Loading +errors occur before server access. + +YAML parsing is strict. The loader accepts one UTF-8 document with a mapping at +the root, and it rejects duplicate keys, aliases, anchors, merge keys, and custom +tags. Only `true` and `false` are booleans, so `off` and `yes` stay strings. +Timestamps stay strings. A document may be at most 1 MiB, 32 levels deep, and +100,000 nodes. Errors point at the source: + +```text +blueprint.yml:24:9 +topology.receiver.vertex_processor.params.redundant_streams +Expected a boolean; received a different type. +``` + +`engine.validate(blueprint)`, or `blueprint.validate_full(registry)`, resolves +every variant, including variants you are not about to apply, and reports +processors that are not registered. Planning only needs the processors for the +scope you selected, so an Inventory-only run works without a topology plugin. + +`BlueprintValidationError.issues` holds `ValidationIssue` records (`path`, +`message`, `code`, `source`, `line`, `column`). Issues carry a path, not the raw +value, so secrets do not leak into errors. + +For editor support, use the packaged JSON Schema from `published_json_schema()`, +or the file `videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json`. +It describes the built-in processors' parameters and accepts any other +`processor_type` with unconstrained `params`, so it also works with processors +registered elsewhere. + +When you use custom processors, export `engine.json_schema()` instead. It limits +`processor_type` to the engine's registered processors (a variant may still omit +it to inherit the `default` processor) and validates each one's `params`, so +editors flag unknown or misspelled processors and invalid parameters. +`Blueprint.json_schema(registry, restrict_processor_types=True)` does the same +without an engine; without the flag, only the parameter schemas are added. + +The schema is an editor aid. Runtime validation (`engine.validate`, `plan`) is +authoritative and also catches what the packaged schema lets through. + +## 6. Naming + +Names are built from the facts you pass in. The engine does not read the label +currently on the server and append to it, so a second apply cannot grow a second +suffix. A rename updates the same server ids. + +| Entry | Default (`DEFAULT_NAMING`) | +|---|---| +| `inventory_label` | `{device.label}` | +| `device_label` | `{device.label}` for device targets. A module target does not rename the parent device. | +| `endpoint_label` | `device-a-TX-video-01`, or `device-a-M1-RX-audio-01` when you set a module position | +| `inventory_description`, `device_description`, `endpoint_description` | Left unmanaged | + +Available fields: + +| Field | Source | +|---|---| +| `device.key`, `device.label`, `device.description` | `BlueprintDevice` | +| `module.position` | `BlueprintDevice.module_position` | +| `endpoint.direction`, `endpoint.media`, `endpoint.index`, `endpoint.engine`, `endpoint.program`, `endpoint.leg` | The processor's `EndpointIdentity` | +| `vertex.factory_label`, `vertex.id` | The discovered vertex | +| `attributes.` | Scalar leaves of `BlueprintDevice.attributes` | + +An expression is a template string, or a tree of blocks: + +| Block | YAML form | Python form | +|---|---|---| +| Text | `"literal {field}"` or `{text: ...}` | `"..."` or `Text("...")` | +| Field | `{field: path, format, mapping, prefix, suffix}` | `Field("path", format=..., mapping=..., prefix=..., suffix=...)` | +| Join | `{join: [...], separator, skip_missing}` | `Join(parts=[...], separator=..., skip_missing=...)` | + +```yaml +naming: + endpoint_label: + join: + - "{device.label}" + - field: module.position + prefix: M + - "{endpoint.direction}" + - "{endpoint.media}" + - "{endpoint.index:02d}" + separator: "-" + skip_missing: true +``` + +```python +from videoipath_automation_tool.blueprints import Field, Join, NamingScheme + +scheme = NamingScheme( + endpoint_label=Join( + parts=[ + Field("device.label"), + "E{endpoint.engine}P{endpoint.program}", + Join(parts=[Field("endpoint.direction"), Field("endpoint.media", mapping={"video": "20", "audio": "30"})]), + Field("endpoint.index", format="02d"), + ], + separator="-", + ) +) +result = engine.apply(device, blueprint, naming=scheme) # device-a-E0P1-TX20-01 +``` + +Format specs cover alignment, zero padding, minimum width, `d`, and `s`. `02d` +sets a minimum width, so `123` stays `123`. + +`skip_missing` skips only an optional value that is absent. An unknown field, a +type error, or a mapping that fails still fails the plan. + +**Layering.** For each entry, later sources replace earlier ones: the library +default, then the engine `naming`, then the blueprint (a topology entry replaces +the same inventory entry), then the `naming` argument on that call. An entry is +replaced as a whole; expressions are not merged. Set an entry to `None` to turn +that name off. `NamingScheme.layered(*schemes)` performs the same combination. + +When the blocks cannot express a name, put a computed value in `attributes`, or +pass a Python object with `render(context: NameContext) -> str` (the +`NameRenderer` protocol) for that entry. Python renderers are not available in +YAML. + +**Checks.** Before any write, every rendered endpoint label is checked. It has to +be non-empty, it cannot contain control characters, and it cannot duplicate +another endpoint label on the same device, including endpoints this plan does not +change. A collision fails and names the colliding ids. +`ApplyOptions(naming_collisions="allow")` lets the write proceed. Names are kept +as rendered. The engine does not truncate them, and it does not append a number +to make them unique. + +## 7. Planning and applying + +```python +from videoipath_automation_tool.blueprints import ApplyOptions + +plan = engine.plan( + device, + blueprint, + scope="all", # "all" | "inventory" | "topology" + inventory_variant="default", + topology_variant="receiver", + options=ApplyOptions(sync="add_only", discovery_timeout=60), +) +``` + +**`ApplyOptions`** + +| Field | Default | Meaning | +|---|---|---| +| `sync` | `"add_only"` | Topology synchronization policy (below). | +| `discovery_timeout` | `30.0` | Seconds to wait for discovery during apply. | +| `poll_interval` | `1.0` | Seconds between discovery polls. | +| `naming_collisions` | `"reject"` | `"allow"` lets duplicate endpoint labels through. | + +Planning reads the current state. It does not write, synchronize, stage snapshot +edits, or create catalog entries. The plan lists every phase with the exact +before and after values (secrets redacted), where each value came from +(`default`, `variant:`, `instance`, `naming`, `processor:`, +`override:`), the interface bindings, and any diagnostics. + +Apply then runs the phases in this order +([ADR-002](./decisions/002-plan-then-apply.md)): + +1. `inventory` creates or updates the record. A new id is stored on the result immediately. +2. `discovery` waits, bounded by `discovery_timeout`. +3. `topology_sync` adds the device to the topology, or synchronizes it. +4. `topology` edits the device and its vertices in one Inspect transaction, with conflict checking. +5. `module_tags` assigns or unassigns tags as separate actions. +6. `verification` reads the device back. `result.verification` is `confirmed` or `unconfirmed`. + +A phase with nothing to do is skipped. If the whole apply would change nothing, +it writes nothing. + +### Deferred topology work + +Sometimes the topology edits depend on work that has not happened yet: a record +this plan creates, Inventory changes that may alter discovery (address, +alternative addresses, credentials, generic or custom settings, `active`), a +device that is not in the topology yet, or a synchronization that is still +pending. The plan marks that work `deferred`, and `plan.fully_resolved` is +`False`. During `apply()` the engine polls for discovery up to +`discovery_timeout`, then computes the exact edits from the blueprint, +parameters, naming, and target captured on the plan. Those edits are recorded on +the result. If that later step fails, the earlier phases stay applied, and the +result says so. + +If you want to see every write before it happens, split the scopes. Apply +`scope="inventory"`, add the device with `app.inspect.add_devices_to_topology([...])`, +then build a fully resolved `scope="topology"` plan, read it, and call +`plan.apply()`. + +### Dry runs + +`plan.apply(dry_run=True)` runs the same stale-plan, conflict, and pending-edit +checks, then returns before any write. Deferred topology stays `deferred`, +because those edits depend on writes the dry run does not perform. `status` is +`planned` when writes would have run, and `no_change` otherwise. You can dry-run a +plan and apply that same plan afterwards. + +### Synchronization + +| `ApplyOptions.sync` | Behavior | +|---|---| +| `"none"` | The device must already be in the topology. The engine does not add it and does not synchronize it. | +| `"add_only"` | Default. Adds the device and synchronizes new elements only. If that synchronization would update or remove elements, the apply fails. It does not escalate to a full reconcile. | +| `"reconcile"` | A full synchronization is allowed. | + +Service conflicts fail the apply. The engine does not invalidate or cancel +services. A module target does not add its parent device to the topology and +does not synchronize it. If the parent is absent, planning fails; if it still +needs a sync, the plan reports a diagnostic. + +### Conflict checks + +Before writing, the engine reads the Inventory record and the scoped topology +again. If anything the plan relied on has changed, it raises +`BlueprintConflictError`. Build a new plan. The Inspect transaction then checks +its own baseline, and module tags are read again immediately before their phase. +These checks run in the client, so a short gap between the read and the write +remains. Uncommitted edits on `app.inspect` that overlap the plan are rejected, +so commit or discard them first. Pending edits outside the plan are left as they +are. + +## 8. Results, errors, and recovery + +`plan.apply()` returns an `ApplyResult` when `status` is `succeeded` or +`no_change` (`planned` on a dry run). On failure it raises `BlueprintApplyError`. +The exception's `.result` has `status` `failed` (nothing was written), `partial` +(something was written), or `unknown` (a write raised and the client cannot prove +the server rejected it). The original error is `__cause__`. + +```python +from videoipath_automation_tool.blueprints import BlueprintApplyError + +try: + result = engine.apply(device, blueprint) +except BlueprintApplyError as error: + result = error.result + if result.inventory_id: # created before the failure: bind it, never create again + save_binding(device.key, result.inventory_id) + raise +``` + +What you do next depends on how far the apply got: + +- The record was created, then discovery timed out. `status` is `partial` and `inventory_id` is set. Retry with `BlueprintDevice(..., inventory_id=result.inventory_id)`. +- Topology committed, then a module tag failed. `status` is `partial`. The result lists the committed edits and the tag operations that finished. Replan to finish the rest. +- A write timed out. `status` is `unknown`. Read the server back before you retry. Building a new plan does that read. +- Verification could not confirm the write, or a later phase failed before the topology read-back ran. The write stays applied, and `verification` is `"unconfirmed"`. + +The engine does not roll a partial apply back. + +All errors derive from `BlueprintError`: + +| Error | Raised when | +|---|---| +| `BlueprintValidationError` | A document, naming expression, or parameter is invalid. `.issues` lists `ValidationIssue`s; `.codes` lists their codes. | +| `BlueprintTargetError` | A binding is missing, ambiguous, or outside the target scope. | +| `ProcessorInputError` | The discovered topology is unsupported or still incomplete. | +| `BlueprintCapabilityError` | The engine cannot perform the requested operation. | +| `BlueprintConflictError` | The plan is stale: the server changed since planning. | +| `BlueprintApplyError` | An apply failed. `.result` is the `ApplyResult`. | + +## 9. Engine configuration + +```python +engine = BlueprintEngine( + app, + processors={"example-org.simple-video": SimpleVideoProcessor}, + options=ApplyOptions(sync="add_only", discovery_timeout=60), + naming=NamingScheme(endpoint_label="{device.label}-{endpoint.direction}-{endpoint.index:02d}"), +) +# equivalent: +engine = BlueprintEngine(app) +engine.register_processor("example-org.simple-video", SimpleVideoProcessor) +engine.configure(options=ApplyOptions(sync="add_only", discovery_timeout=60)) +engine.configure(naming=NamingScheme(endpoint_label="{device.label}-{endpoint.direction}-{endpoint.index:02d}")) +``` + +Constructing the engine does not touch the app or the network. In `configure`, +an omitted argument keeps the current default, a supplied model replaces it, and +`None` restores the library default. Each plan keeps its own copy of the options, +naming, and processor registrations that were in effect when you built it. +Configuration you change afterwards applies to later plans only. Options and +naming passed to `plan()` apply to that plan only. Two engines can share one app +and still keep separate registries. Sharing the app does not make concurrent +writes to the same server safe. + +Engine logs go to the `videoipath_automation_tool_blueprints` logger unless you +pass `logger=`. + +## 10. Source-system adapters + +The library does not depend on NetBox, or on any other source system. An adapter +is an ordinary function you write: + +```python +from videoipath_automation_tool.blueprints import BlueprintDevice, Credentials, ModuleTarget + + +def to_blueprint_device(record: dict) -> BlueprintDevice: + """Translate one source record (already selected and policy-checked by the caller).""" + return BlueprintDevice( + key=str(record["id"]), + label=record["name"], + inventory_id=record.get("videoipath_inventory_id"), + management_address=record["primary_address"], + alternative_addresses=record.get("alternative_addresses"), + credentials=Credentials(username=record["user"], password=fetch_secret(record)), + topology=ModuleTarget(device_id=record["parent_inspect_id"], module_id=record["inspect_module_id"]) + if record.get("inspect_module_id") + else None, + module_position=record.get("module_position"), + attributes={"site": record.get("site")}, + ) +``` + +You keep the decisions around the engine: which blueprint and variant to use, +which address role is allowed, where credentials come from, where returned ids +are stored, how cables are built, and when the job runs. + +## 11. Limits and verification status + +- An Inventory round-trip keeps the fields the Inventory models know about, the same way `InventoryApp.update_device` does. Wire fields those models do not parse are dropped. +- `http_auth` is the raw server mode code. Alternative addresses that carry credentials use the `{address, authentication: {user, password}}` entry shape already used by Inventory. +- Tag references are exact catalog ids. The client does not check that the tag exists. +- Endpoint label uniqueness is checked per device, not across the whole system. +- Vertex control configuration waits until its Inspect mapping has been verified. +- Offline tests cover the feature with synthetic Inspect and Inventory data. Try the writes you care about on a test system before you use them in production. +- Also outside this feature: picking a processor from the driver automatically, pruning old configuration, rolling back a partial apply, migrating drivers, inferring cables or edges, provisioning services, and scheduling a fleet. + +## 12. Public API index + +Everything below is importable from `videoipath_automation_tool.blueprints`. + +| Area | Names | +|---|---| +| Engine | `BlueprintEngine`, `BlueprintPlan`, `BlueprintApp` | +| Document | `Blueprint`, `InventorySettings`, `CatalogId`, `published_json_schema` | +| Device facts | `BlueprintDevice`, `Credentials`, `AlternativeAddress`, `DeviceTarget`, `ModuleTarget` | +| Patches | `DevicePatch`, `ModulePatch`, `VertexPatch`, `Coordinates`, `TagDelta` | +| Options and results | `ApplyOptions`, `ApplyResult`, `PhaseResult`, `PlannedPhase`, `PlannedOperation`, `FieldChange`, `InterfaceBinding`, `Diagnostic` | +| Naming | `NamingScheme`, `DEFAULT_NAMING`, `Text`, `Field`, `Join`, `NameContext`, `NameRenderer` | +| Processors | `VertexProcessor`, `ProcessorRegistry`, `ProcessingContext`, `ProcessorResult`, `VertexEdit`, `EndpointIdentity`, `SourceFacts`, `DriverContext`, `DeviceRecord`, `ModuleRecord`, `PortRecord`, `VertexRecord`, `MatroxConvertIPProcessor`, `MatroxConvertIPParams` | +| Errors | `BlueprintError`, `BlueprintValidationError`, `ValidationIssue`, `BlueprintTargetError`, `ProcessorInputError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | diff --git a/docs/examples/07_blueprints/01_onboard_with_blueprint.py b/docs/examples/07_blueprints/01_onboard_with_blueprint.py new file mode 100644 index 0000000..82c657b --- /dev/null +++ b/docs/examples/07_blueprints/01_onboard_with_blueprint.py @@ -0,0 +1,79 @@ +"""Onboard and configure a device from a blueprint. + +Description +----------- +Loads ``matrox-convertip.yml``, describes one device with neutral instance facts, previews the plan, +and applies it (as a dry run by default). On the first run the Inventory record is created and the topology work is deferred +until the driver has discovered the device; later runs bind the stored Inventory id and are no-ops +when nothing changed. A partial failure still reports the created id so a retry never creates a +duplicate. + +Prerequisites +------------- +- A reachable VideoIPath server (VideoIPath >= 2025.4) with Inventory and topology write access. + +Related examples +---------------- +- 02_custom_processor_and_naming.py +- 06_workflows/01_full_onboarding_pipeline.py +""" + +from __future__ import annotations + +from pathlib import Path + +from videoipath_automation_tool import VideoIPathApp +from videoipath_automation_tool.blueprints import ( + ApplyOptions, + BlueprintApplyError, + BlueprintDevice, + BlueprintEngine, + Credentials, +) + +SERVER_ADDRESS = "" +USERNAME = "" +PASSWORD = "" + +BLUEPRINT = Path(__file__).with_name("matrox-convertip.yml") +DRY_RUN = True # set to False to write to the server +STORED_INVENTORY_ID: str | None = None # e.g. "device12" once known (persist it in your source system) + + +def main() -> None: + # --- 1. Connect and build the engine --------------------------------------- + app = VideoIPathApp(server_address=SERVER_ADDRESS, username=USERNAME, password=PASSWORD, use_https=False) + engine = BlueprintEngine(app, options=ApplyOptions(sync="add_only", discovery_timeout=60)) + + # --- 2. Describe the device with neutral facts ----------------------------- + device = BlueprintDevice( + key="device-a", + label="device-a", + description="studio-a", + inventory_id=STORED_INVENTORY_ID, + management_address="192.0.2.10", + credentials=Credentials(username="test-user", password="test-password"), + ) + + # --- 3. Load the blueprint internally and preview -------------------------- + plan = engine.plan(device, BLUEPRINT, topology_variant="receiver") + print(plan.summary()) + if not plan.has_changes: + return + + # --- 4. Apply and handle partial outcomes ---------------------------------- + try: + result = plan.apply(dry_run=DRY_RUN) # a dry run checks everything but writes nothing + except BlueprintApplyError as error: + result = error.result + print(f"Apply {result.status}; known Inventory id: {result.inventory_id}") + for phase in result.phases: + print(f" {phase.name}: {phase.status} {phase.message or ''}") + raise + print(f"{result.status}: inventory={result.inventory_id} verification={result.verification}") + for binding in result.interface_bindings: + print(f" {binding.key} -> {binding.port_id}") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/07_blueprints/02_custom_processor_and_naming.py b/docs/examples/07_blueprints/02_custom_processor_and_naming.py new file mode 100644 index 0000000..3ab0837 --- /dev/null +++ b/docs/examples/07_blueprints/02_custom_processor_and_naming.py @@ -0,0 +1,114 @@ +"""Register a custom vertex processor and use a custom naming convention. + +Description +----------- +Defines a tiny processor for a single-sender device, registers it on one engine, and applies a +topology-only blueprint (written as a dictionary) to an existing Inspect device. The endpoint label +convention (``-EP--``) is a ``NamingScheme``, +so it can change without touching the processor. Also shows a module-scoped call. + +Prerequisites +------------- +- A reachable VideoIPath server (VideoIPath >= 2025.4) with topology write access. +- An Inspect device whose codec sender port is labelled ``Video Sender``. + +Related examples +---------------- +- 01_onboard_with_blueprint.py +- 03_topology_and_inspect/02_configure_vertices_inspect.py +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from videoipath_automation_tool import VideoIPathApp +from videoipath_automation_tool.blueprints import ( + Blueprint, + BlueprintDevice, + BlueprintEngine, + DeviceTarget, + EndpointIdentity, + Field, + Join, + ModuleTarget, + NamingScheme, + ProcessingContext, + ProcessorResult, + VertexEdit, + VertexPatch, + VertexProcessor, +) + +SERVER_ADDRESS = "" +USERNAME = "" +PASSWORD = "" + +DEVICE_ID = "device12" # existing Inspect device +MODULE_ID: str | None = None # e.g. "device12.dev.1" to configure one module only +DRY_RUN = True # set to False to write to the server + + +class SimpleParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + factory_label: str + + +class SimpleVideoProcessor(VertexProcessor[SimpleParams]): + params_model = SimpleParams + + def process(self, context: ProcessingContext, params: SimpleParams) -> ProcessorResult: + vertex = context.require_vertex(kind="codec", factory_label=params.factory_label, vertex_type="In") + return ProcessorResult( + vertices=[ + VertexEdit( + vertex_id=vertex.id, + fields=VertexPatch(use_as_endpoint=True, description=params.factory_label), + endpoint=EndpointIdentity(direction="TX", media="video", index=1, engine=0, program=1), + ) + ] + ) + + +NAMING = NamingScheme( + endpoint_label=Join( + parts=[ + Field("device.label"), + "E{endpoint.engine}P{endpoint.program}", + Join(parts=[Field("endpoint.direction"), Field("endpoint.media", mapping={"video": "20", "audio": "30"})]), + Field("endpoint.index", format="02d"), + ], + separator="-", + ) +) + +BLUEPRINT = { + "schema_version": 1, + "topology": { + "default": { + "vertex_processor": { + "processor_type": "example-org.simple-video", + "params": {"factory_label": "Video Sender"}, + } + } + }, +} + + +def main() -> None: + app = VideoIPathApp(server_address=SERVER_ADDRESS, username=USERNAME, password=PASSWORD, use_https=False) + engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}, naming=NAMING) + + target = ModuleTarget(device_id=DEVICE_ID, module_id=MODULE_ID) if MODULE_ID else DeviceTarget(device_id=DEVICE_ID) + device = BlueprintDevice( + key="device-a", label="device-a", topology=target, module_position="1" if MODULE_ID else None + ) + + plan = engine.plan(device, Blueprint.from_dict(BLUEPRINT), scope="topology") + print(plan.summary()) # e.g. label: '' -> 'device-a-E0P1-TX20-01' + if plan.has_changes: + print(plan.apply(dry_run=DRY_RUN).status) + + +if __name__ == "__main__": + main() diff --git a/docs/examples/07_blueprints/matrox-convertip.yml b/docs/examples/07_blueprints/matrox-convertip.yml new file mode 100644 index 0000000..b07f73e --- /dev/null +++ b/docs/examples/07_blueprints/matrox-convertip.yml @@ -0,0 +1,41 @@ +# Example blueprint for Matrox ConvertIP devices (synthetic tag ids). +# Validate offline: BlueprintEngine(app).validate(path) +schema_version: 1 + +inventory: + default: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + generic_settings: + enable_https: false + http_auth: 0 + custom_settings: + port: 8080 + disable_rx_sdp_with_null: true + indices_in_ids: false + +topology: + default: + device: + icon_size: medium + ip_vertex_mapping: + stream-a: [P1] + stream-b: [P2] + vertex_processor: + processor_type: matrox.convertip.default + params: + redundant_streams: true + video_receiver_tags: [Format~~video-tag-a] + video_sender_tags: [Format~~video-tag-b] + audio_receiver_tags: [Format~~audio-tag-a] + audio_sender_tags: [Format~~audio-tag-b] + + receiver: + vertex_processor: + params: + mode: rx + + four-split: + vertex_processor: + params: + mode: four_split + video_receiver_tags: [Format~~video-tag-c] diff --git a/docs/examples/README.md b/docs/examples/README.md index b9abe67..d2cc12c 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -94,3 +94,14 @@ Each scenario is implemented twice — once with the modern **Inspect** app, onc report (safe to run against production). - [03_bulk_retag_and_relabel.py](06_workflows/03_bulk_retag_and_relabel.py) — apply a naming/tagging policy across a fleet with a dry-run report and a single batched commit. + +### 07 — Blueprints + +See the [Blueprint guide](../getting-started-guide/05_Blueprints.md). + +- [matrox-convertip.yml](07_blueprints/matrox-convertip.yml) — example blueprint with Inventory + settings, interface mapping, the Matrox processor, and `receiver` / `four-split` variants. +- [01_onboard_with_blueprint.py](07_blueprints/01_onboard_with_blueprint.py) — preview and apply a + blueprint (creation with deferred discovery, idempotent re-runs, partial-failure handling). +- [02_custom_processor_and_naming.py](07_blueprints/02_custom_processor_and_naming.py) — register a + custom vertex processor and a custom endpoint naming convention; device or module scope. diff --git a/docs/getting-started-guide/05_Blueprints.md b/docs/getting-started-guide/05_Blueprints.md new file mode 100644 index 0000000..e68dae6 --- /dev/null +++ b/docs/getting-started-guide/05_Blueprints.md @@ -0,0 +1,188 @@ +# 05 — Blueprint-driven Configuration + +A **blueprint** is a YAML file that describes how one *kind* of device should be +configured: its Inventory driver settings, how it appears in Inspect, and which of +its discovered vertices become named, tagged endpoints. You write it once and +apply it to every real device of that kind. + +This page covers the essentials. The complete reference (every field, option, +error, and the API index) is in +[docs/architecture/blueprints](../architecture/blueprints/README.md). + +## 1. Key concepts + +| Concept | What it is | +|---|---| +| `Blueprint` | The YAML document: an `inventory` and/or `topology` section, each with a `default` entry and optional named **variants**. | +| `BlueprintDevice` | The facts of one real device that only your source system knows: id, label, addresses, credentials. | +| `BlueprintEngine` | Combines a blueprint with a device. `plan()` previews the changes, `apply()` writes them through `app.inventory` and `app.inspect`. | +| Vertex processor | Plugin code that recognizes a driver's discovered vertices and proposes endpoint settings. A Matrox ConvertIP processor is built in. | +| Naming | Rules that build labels from device facts, so re-applying never stacks suffixes. | + +Two rules shape everything else: + +- **Plan, then apply.** Planning only reads. The plan shows every change with + before and after values, so you can review it before anything is written. +- **Only what you set is managed.** Fields the blueprint leaves out stay as they + are on the server. Removing a field from the blueprint does not reset it. + +## 2. Quick start + +```python +from videoipath_automation_tool import VideoIPathApp +from videoipath_automation_tool.blueprints import BlueprintDevice, BlueprintEngine + +app = VideoIPathApp() +engine = BlueprintEngine(app) # the app has no blueprint attribute; create the engine yourself + +device = BlueprintDevice(key="device-a", label="device-a", management_address="192.0.2.10") + +plan = engine.plan(device, "blueprints/matrox-convertip.yml") +print(plan.summary()) # review the changes + +result = plan.apply() +print(result.status, result.inventory_id) # persist inventory_id in your source system +``` + +`engine.apply(device, blueprint)` plans and applies in one call. Pass +`dry_run=True` to `apply()` to run every check without writing. The blueprint +argument is a file path or a `Blueprint` object; PyYAML ships with the package. + +On later runs, pass the stored id as `BlueprintDevice(..., inventory_id=...)` so the +engine updates that record instead of creating a new one. When nothing changed, +the apply writes nothing. + +## 3. Writing a blueprint + +```yaml +schema_version: 1 + +inventory: + default: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + generic_settings: {enable_https: false, http_auth: 0} + custom_settings: {port: 8080} + +topology: + default: + device: {icon_size: medium} + vertex_processor: + processor_type: matrox.convertip.default + params: + redundant_streams: true + video_sender_tags: [Format~~video-tag-b] + receiver: + vertex_processor: + params: {mode: rx} +``` + +- **`inventory`** sets the driver and its settings. `custom_settings` are checked + against the driver schema, so a typo or wrong type fails the plan before any write. +- **`topology`** sets device appearance, the vertex processor and its `params`, + and optional explicit vertex overrides. +- **Variants** such as `receiver` are patches on top of `default`. Choose them per + call with `inventory_variant=` and `topology_variant=`. +- **Tags** are exact catalog ids. A list replaces the local tags; `{add: [...], remove: [...]}` + changes only the listed ones. + +Check every variant offline with `engine.validate("blueprints/matrox-convertip.yml")`. +Errors point at the file, line, and field. For editor completion, use the +packaged JSON Schema (`published_json_schema()`), or `engine.json_schema()` when +you register custom processors (it only accepts registered `processor_type` values). + +Details: [document reference](../architecture/blueprints/reference.md#3-blueprint-document), +[variants](../architecture/blueprints/reference.md#4-variants), +[loading and validation](../architecture/blueprints/reference.md#5-loading-and-validation). + +## 4. Describing a device + +```python +from videoipath_automation_tool.blueprints import BlueprintDevice, Credentials, ModuleTarget + +device = BlueprintDevice( + key="source-id-42", # your id; never written to VideoIPath + label="device-a", + inventory_id="device12", # omit on first run to create the record + management_address="192.0.2.10", + credentials=Credentials(username="test-user", password="test-password"), + attributes={"site": "site-a"}, # extra facts for naming +) +``` + +Credentials stay out of YAML and are redacted in plans, results, and errors. To +configure one module of a chassis, pass `topology=ModuleTarget(device_id, module_id)` +with the exact Inspect ids. + +Details: [all fields and supported arrangements](../architecture/blueprints/reference.md#2-describing-a-device-blueprintdevice). + +## 5. Naming + +By default, Inventory and device labels are the device `label`, and endpoint +labels look like `device-a-TX-video-01` (or `device-a-M1-RX-audio-01` with a +module position). Override any entry in the blueprint or per call: + +```yaml +topology: + default: + naming: + endpoint_label: "{device.label}-{endpoint.direction}-{endpoint.index:02d}" +``` + +Endpoint labels must be unique on the device; a collision fails the plan. + +Details: [fields, `Text` / `Field` / `Join` blocks, layering](../architecture/blueprints/reference.md#6-naming). + +## 6. What apply does + +Apply runs these phases in order, skipping any with nothing to do: + +1. **inventory** — create or update the record. +2. **discovery** — wait for the driver to discover the device (`discovery_timeout`). +3. **topology_sync** — add the device to the topology or synchronize it. +4. **topology** — commit device and vertex edits in one Inspect transaction. +5. **module_tags** — assign or remove module tags. +6. **verification** — read back what was written. + +When the plan creates the Inventory record, the topology edits cannot be known +yet. The plan marks them `deferred` and computes them during apply, after +discovery. If the server changed since planning, apply raises +`BlueprintConflictError`; build a new plan. Behavior is tuned with +`ApplyOptions` (`sync`, `discovery_timeout`, ...). + +Details: [planning and applying](../architecture/blueprints/reference.md#7-planning-and-applying). + +## 7. Handling failures + +Apply never rolls back. If a later phase fails, the earlier ones stay applied and +`BlueprintApplyError.result` tells you how far it got: + +```python +from videoipath_automation_tool.blueprints import BlueprintApplyError + +try: + result = engine.apply(device, "blueprints/matrox-convertip.yml") +except BlueprintApplyError as error: + if error.result.inventory_id: # record was created: store it so a retry updates, not duplicates + save_binding(device.key, error.result.inventory_id) + raise +``` + +Details: [results, errors, and recovery](../architecture/blueprints/reference.md#8-results-errors-and-recovery). + +## 8. Custom processors + +When no built-in processor fits a driver, write one: a `VertexProcessor` subclass +with a Pydantic `params_model` that reads the discovered vertices and returns +proposed edits. Register it on the engine and reference its id in YAML: + +```python +engine.register_processor("example-org.simple-video", SimpleVideoProcessor) +``` + +Details: [processor contract and the Matrox built-in](../architecture/blueprints/processors.md). + +## Next steps + +- Runnable scripts: [docs/examples/07_blueprints](../examples/07_blueprints/) +- Full reference: [reference.md](../architecture/blueprints/reference.md) +- Design and decisions: [concepts.md](../architecture/blueprints/concepts.md) diff --git a/docs/getting-started-guide/README.md b/docs/getting-started-guide/README.md index 8b26c12..34c7d03 100644 --- a/docs/getting-started-guide/README.md +++ b/docs/getting-started-guide/README.md @@ -25,6 +25,7 @@ onboarding is unchanged and remains a prerequisite for both. - A. [Topology App](03_A_Topology.md) (classic / legacy) - B. [Inspect App](03_B_Inspect.md) (recommended on 2025.4+) 4. [Configuring Multicast Pools](04_Multicast_Pools.md) +5. [Blueprint-driven Configuration](05_Blueprints.md) — reusable YAML blueprints, planning, and custom vertex processors For runnable, task-oriented scripts covering realistic automation scenarios (including paired Topology/Inspect examples), see the diff --git a/poetry.lock b/poetry.lock index 9416f46..6ca195f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -734,7 +734,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -927,4 +927,4 @@ python-discovery = ">=1.6" [metadata] lock-version = "2.1" python-versions = ">=3.11" -content-hash = "5c500d306e5ade2dde4b17e70968229706b52b740623241564cf5b709853b3f6" +content-hash = "5bd868328626eba09185cfa6e39d2629eb2dfa4bf34a7c6bd209df8c9b6f329e" diff --git a/pyproject.toml b/pyproject.toml index d61594b..f2a4e53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "pydantic-settings (>=2.15.0,<3.0.0)", "urllib3 (>=2.8.0,<3.0.0)", "deepdiff (>=8.6.2,<9.0.0)", + "pyyaml (>=6.0.1,<7.0.0)", ] [project.urls] diff --git a/src/videoipath_automation_tool/apps/inventory/app/app.py b/src/videoipath_automation_tool/apps/inventory/app/app.py index 9061952..9676529 100644 --- a/src/videoipath_automation_tool/apps/inventory/app/app.py +++ b/src/videoipath_automation_tool/apps/inventory/app/app.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Iterable from typing import List, Literal, Optional from typing_extensions import deprecated @@ -231,6 +232,13 @@ def find_device_id_by_label( else: raise ValueError(f"Invalid label_search_mode: {label_search_mode}") + def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> list[str]: + """Device ids whose management or alternate address matches any of ``addresses``. + + One server read for every candidate spelling. An empty inventory response is no match. + """ + return self._inventory_api.find_device_ids_by_addresses(addresses) + def list_device_ids_by_driver(self, driver: DriverLiteral) -> List[str]: """Method to list all device ids by driver id. diff --git a/src/videoipath_automation_tool/apps/inventory/inventory_api.py b/src/videoipath_automation_tool/apps/inventory/inventory_api.py index 0a18464..60ceecd 100644 --- a/src/videoipath_automation_tool/apps/inventory/inventory_api.py +++ b/src/videoipath_automation_tool/apps/inventory/inventory_api.py @@ -1,6 +1,7 @@ import logging import time import urllib.parse +from collections.abc import Iterable from typing import List, Optional, Type from uuid import uuid4 @@ -575,6 +576,35 @@ def get_device_id_by_address(self, address: str, include_alt_addresses: bool = T else: return device_ids + def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> list[str]: + """Device ids whose management or alternate address equals any of ``addresses``. + + One bulk read. A missing item list is no match. A missing ``altAddresses`` list on a device + is treated as empty. + """ + wanted = {address for address in addresses if address} + if not wanted: + return [] + url = "/rest/v2/data/config/devman/devices/*/config/cinfo/address,altAddresses,altAddresses/**" + response = self.vip_connector.rest.get(url) + devices = ( + response.data.get("config", {}).get("devman", {}).get("devices", {}).get("_items") + if isinstance(response.data, dict) + else None + ) + if not isinstance(devices, list): + return [] + found: set[str] = set() + for device in devices: + cinfo = device.get("config", {}).get("cinfo", {}) + known = {cinfo.get("address")} + alternates = cinfo.get("altAddresses") or [] + if isinstance(alternates, list): + known.update(alternates) + if known & wanted and isinstance(device.get("_id"), str): + found.add(device["_id"]) + return sorted(found) + def get_device_id_by_meta_field_value(self, meta_field: str, value: str) -> Optional[str | List[str]]: """Method to get a device id by given meta field value from VideoIPath-Inventory diff --git a/src/videoipath_automation_tool/blueprints/__init__.py b/src/videoipath_automation_tool/blueprints/__init__.py new file mode 100644 index 0000000..c69f71c --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/__init__.py @@ -0,0 +1,128 @@ +"""Blueprint-driven Inventory and Inspect configuration. + +A standalone feature: construct :class:`BlueprintEngine` with an app (``BlueprintEngine(app)``), +describe a device with :class:`BlueprintDevice`, then pass a YAML file path or :class:`Blueprint` to +``engine.plan(...)`` (read-only preview) and ``plan.apply()``, or ``engine.apply(...)`` in one call. +``VideoIPathApp`` neither imports nor exposes this package. + +Importing this package and loading blueprints never connects to a server. +""" + +from __future__ import annotations + +from videoipath_automation_tool.blueprints.engine import BlueprintApp, BlueprintEngine, BlueprintPlan +from videoipath_automation_tool.blueprints.errors import ( + BlueprintApplyError, + BlueprintCapabilityError, + BlueprintConflictError, + BlueprintError, + BlueprintTargetError, + BlueprintValidationError, + ProcessorInputError, + ValidationIssue, +) +from videoipath_automation_tool.blueprints.models import ( + AlternativeAddress, + ApplyOptions, + ApplyResult, + Blueprint, + BlueprintDevice, + CatalogId, + Coordinates, + Credentials, + DevicePatch, + DeviceTarget, + Diagnostic, + EndpointIdentity, + FieldChange, + InterfaceBinding, + InventorySettings, + ModulePatch, + ModuleTarget, + PhaseResult, + PlannedOperation, + PlannedPhase, + ProcessorResult, + TagDelta, + VertexEdit, + VertexPatch, +) +from videoipath_automation_tool.blueprints.naming import ( + DEFAULT_NAMING, + Field, + Join, + NameContext, + NameRenderer, + NamingScheme, + Text, +) +from videoipath_automation_tool.blueprints.processors import ( + DeviceRecord, + DriverContext, + ModuleRecord, + PortRecord, + ProcessingContext, + ProcessorRegistry, + SourceFacts, + VertexProcessor, + VertexRecord, +) +from videoipath_automation_tool.blueprints.processors.matrox import MatroxConvertIPParams, MatroxConvertIPProcessor +from videoipath_automation_tool.blueprints.resolution import published_json_schema + +__all__ = [ + "DEFAULT_NAMING", + "AlternativeAddress", + "ApplyOptions", + "ApplyResult", + "Blueprint", + "BlueprintApp", + "BlueprintApplyError", + "BlueprintCapabilityError", + "BlueprintConflictError", + "BlueprintDevice", + "BlueprintEngine", + "BlueprintError", + "BlueprintPlan", + "BlueprintTargetError", + "BlueprintValidationError", + "CatalogId", + "Coordinates", + "Credentials", + "DevicePatch", + "DeviceRecord", + "DeviceTarget", + "Diagnostic", + "DriverContext", + "EndpointIdentity", + "Field", + "FieldChange", + "InterfaceBinding", + "InventorySettings", + "Join", + "MatroxConvertIPParams", + "MatroxConvertIPProcessor", + "ModulePatch", + "ModuleRecord", + "ModuleTarget", + "NameContext", + "NameRenderer", + "NamingScheme", + "PhaseResult", + "PlannedOperation", + "PlannedPhase", + "PortRecord", + "ProcessingContext", + "ProcessorInputError", + "ProcessorRegistry", + "ProcessorResult", + "SourceFacts", + "TagDelta", + "Text", + "ValidationIssue", + "VertexEdit", + "VertexPatch", + "VertexProcessor", + "VertexRecord", + "published_json_schema", +] diff --git a/src/videoipath_automation_tool/blueprints/engine.py b/src/videoipath_automation_tool/blueprints/engine.py new file mode 100644 index 0000000..3a2e617 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/engine.py @@ -0,0 +1,959 @@ +"""The standalone :class:`BlueprintEngine`: configuration, read-only planning, and execution. + +``engine.plan(...)`` reads VideoIPath state and returns an immutable :class:`BlueprintPlan`; it never +writes, synchronizes, stages snapshot edits, or creates catalog entries. ``plan.apply()`` executes +exactly that plan through the shared executor, checking the captured baselines first. +``engine.apply(...)`` is ``engine.plan(...).apply()``. ``dry_run=True`` runs the same execution path — +including the stale-plan, conflict, and staged-edit checks — but performs no write. + +Execution phases (absent or unchanged phases are skipped): Inventory → discovery readiness → +topology membership/synchronization → topology commit (one ``InspectTransaction``) → module tags +(separate RPCs) → verification. Inventory and Inspect writes are separate operations; there is no +transaction spanning the whole deployment and no automatic rollback. Failures raise +:class:`BlueprintApplyError` carrying an :class:`ApplyResult` with every known id and phase outcome. +""" + +from __future__ import annotations + +import contextlib +import logging +import time +from collections.abc import Callable, Iterator, Mapping +from os import PathLike +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, PrivateAttr + +from videoipath_automation_tool.apps.inspect.errors import InspectCommitConflictError, InspectCommitError +from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice +from videoipath_automation_tool.blueprints.errors import ( + BlueprintApplyError, + BlueprintCapabilityError, + BlueprintConflictError, + BlueprintError, + BlueprintTargetError, + BlueprintValidationError, + ProcessorInputError, +) +from videoipath_automation_tool.blueprints.inspect import ( + InspectGateway, + TopologyWork, + compute_topology_work, + name_context, + verify_topology, +) +from videoipath_automation_tool.blueprints.inventory import ( + InventoryGateway, + InventoryWork, + build_candidate, + check_baseline, + new_device, + plan_inventory, + recheck_conflicts, + verify, +) +from videoipath_automation_tool.blueprints.models import ( + ApplyOptions, + ApplyResult, + Blueprint, + BlueprintDevice, + DeviceTarget, + Diagnostic, + InterfaceBinding, + ModuleTarget, + PhaseName, + PhaseResult, + PlannedOperation, + PlannedPhase, + Scope, + TopologyTarget, +) +from videoipath_automation_tool.blueprints.naming import DEFAULT_NAMING, NamingScheme, render_name +from videoipath_automation_tool.blueprints.processors import ( + DriverContext, + ProcessorRegistry, + SourceFacts, + VertexProcessor, +) +from videoipath_automation_tool.blueprints.resolution import ResolvedBlueprint, resolve_blueprint +from videoipath_automation_tool.validators.device_id import validate_device_id + +# Sentinel for "argument not supplied" (distinct from an explicit None). +_UNSET: Any = object() + + +class BlueprintApp(Protocol): + """The app interface the engine needs; :class:`VideoIPathApp` satisfies it unchanged.""" + + @property + def inventory(self) -> Any: ... + + @property + def inspect(self) -> Any: ... + + +class BlueprintPlan(BaseModel): + """A reviewed, immutable preview of intended changes. ``apply()`` executes exactly this plan. + + ``fully_resolved`` is ``False`` when topology work depends on earlier phases (Inventory creation, + topology-affecting Inventory changes, topology membership, or pending synchronization); that work + is materialized during ``apply()`` from the captured configuration and recorded in the result. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + source_key: str + scope: Scope + inventory_variant: str | None = None + topology_variant: str | None = None + blueprint_digest: str + driver_id: str | None = None + driver_schema_version: str | None = None + processor_type: str | None = None + inventory_id: str | None = None + topology_target: TopologyTarget | None = None + phases: list[PlannedPhase] + interface_bindings: list[InterfaceBinding] + diagnostics: list[Diagnostic] + skipped_sections: dict[str, str] + fully_resolved: bool + + _captured: _Captured = PrivateAttr() + _executor: _Executor = PrivateAttr() + + @property + def has_changes(self) -> bool: + return any(phase.status in ("planned", "deferred") for phase in self.phases) + + def phase(self, name: PhaseName) -> PlannedPhase | None: + return next((phase for phase in self.phases if phase.name == name), None) + + def apply(self, *, dry_run: bool = False) -> ApplyResult: + """Check the captured assumptions and execute this plan (no configuration overrides). + + With ``dry_run=True`` every check runs against current server state, but no write, topology + action, or tag action is performed; the result reports the writes that would run + (``status="planned"``). Deferred topology work needs earlier writes and is reported as + ``deferred``. + """ + return self._executor.execute(self, dry_run=dry_run) + + def summary(self) -> str: + """Human-readable, redacted summary with before/after values and unresolved work.""" + header = f"Blueprint plan for '{self.source_key}' (scope={self.scope}" + if self.inventory_variant: + header += f", inventory={self.inventory_variant}" + if self.topology_variant: + header += f", topology={self.topology_variant}" + lines = [header + f", digest={self.blueprint_digest[:12]})"] + for phase in self.phases: + line = f"{phase.name} [{phase.status}]" + if phase.reason: + line += f": {phase.reason}" + lines.append(line) + for operation in phase.operations: + lines.append(f" {operation.action} {operation.entity_kind} {operation.entity_id or '(new)'}") + for change in operation.changes: + lines.append(f" ~ {change.field}: {change.before!r} -> {change.after!r} ({change.source})") + for binding in self.interface_bindings: + lines.append(f"interface {binding.key} -> port {binding.port_id} ({binding.candidate})") + for diagnostic in self.diagnostics: + lines.append(f"[{diagnostic.level}] {diagnostic.code}: {diagnostic.message}") + if not self.fully_resolved: + lines.append( + "Not fully resolved: deferred topology work is materialized during apply() after earlier phases; " + "if it then fails validation, earlier phases remain applied and are reported." + ) + return "\n".join(lines) + + def __repr__(self) -> str: + statuses = ", ".join(f"{phase.name}={phase.status}" for phase in self.phases) + return f"BlueprintPlan(source_key={self.source_key!r}, {statuses}, fully_resolved={self.fully_resolved})" + + __str__ = __repr__ + + +class BlueprintEngine: + """Standalone blueprint engine bound to one injected app for its lifetime. + + Only ``app`` is required. ``processors`` (id → class) are added to the built-ins, ``options`` and + ``naming`` set engine-wide defaults; each has a post-initialization equivalent + (:meth:`register_processor`, :meth:`configure`) with identical validation. Construction does + not access app properties or the network. Every plan captures copies of its effective options, + naming, and processor registrations; later configuration only affects future plans. + """ + + def __init__( + self, + app: BlueprintApp, + *, + processors: Mapping[str, type[VertexProcessor[Any]]] | None = None, + options: ApplyOptions | None = None, + naming: NamingScheme | None = None, + logger: logging.Logger | None = None, + ) -> None: + self._app = app + self._logger = logger or logging.getLogger("videoipath_automation_tool_blueprints") + self._registry = ProcessorRegistry(processors) + self._options = ApplyOptions() + self._naming: NamingScheme | None = None + self.configure( + options=options if options is not None else _UNSET, + naming=naming if naming is not None else _UNSET, + ) + self._clock: Callable[[], float] = time.monotonic + self._sleep: Callable[[float], None] = time.sleep + + # --- Configuration --- + + @property + def app(self) -> BlueprintApp: + return self._app + + @property + def options(self) -> ApplyOptions: + return self._options + + @property + def naming(self) -> NamingScheme | None: + return self._naming + + @property + def registry(self) -> ProcessorRegistry: + """A copy of this engine's registrations (mutating it does not affect the engine).""" + return self._registry.copy() + + def register_processor(self, processor_id: str, processor_cls: type[VertexProcessor[Any]]) -> None: + """Register a processor for this engine only; duplicate ids fail.""" + self._registry.register(processor_id, processor_cls) + + def configure( + self, + *, + options: ApplyOptions | None = _UNSET, + naming: NamingScheme | None = _UNSET, + ) -> None: + """Replace engine defaults. Omitted arguments keep the current defaults; a supplied model + replaces that category; an explicit ``None`` resets it to the library defaults. Values are + validated before anything changes.""" + if options is not _UNSET and options is not None and not isinstance(options, ApplyOptions): + raise TypeError("options must be ApplyOptions or None.") + if naming is not _UNSET and naming is not None and not isinstance(naming, NamingScheme): + raise TypeError("naming must be a NamingScheme or None.") + if options is not _UNSET: + self._options = options.model_copy() if options is not None else ApplyOptions() + if naming is not _UNSET: + self._naming = naming.model_copy() if naming is not None else None + + def validate(self, blueprint: Blueprint | str | PathLike[str]) -> None: + """Validate every variant of ``blueprint`` against this engine's processors (offline). + + Accepts a Blueprint instance or a YAML file path (string or path-like object). + + Raises: + BlueprintValidationError: listing all issues, including unregistered processors. + """ + self._load_blueprint(blueprint).validate_full(self._registry) + + def json_schema(self) -> dict[str, Any]: + """Blueprint JSON Schema for this engine: ``processor_type`` is limited to the registered + processors, and each one's ``params`` are described by its parameter schema.""" + return Blueprint.json_schema(registry=self._registry, restrict_processor_types=True) + + # --- Planning and application --- + + def plan( + self, + device: BlueprintDevice, + blueprint: Blueprint | str | PathLike[str], + *, + scope: Scope = "all", + inventory_variant: str = "default", + topology_variant: str = "default", + naming: NamingScheme | None = None, + options: ApplyOptions | None = None, + ) -> BlueprintPlan: + """Load/resolve the blueprint, read current state, and compute changes. Read-only. + + ``blueprint`` accepts a Blueprint instance or a YAML file path (string or path-like + object). Files are loaded once during planning; applying the plan never re-reads them. + Use ``Blueprint.from_yaml(text)`` for raw YAML strings. + """ + if not isinstance(device, BlueprintDevice): + raise TypeError("device must be a BlueprintDevice.") + if naming is not None and not isinstance(naming, NamingScheme): + raise TypeError("naming must be a NamingScheme.") + if options is not None and not isinstance(options, ApplyOptions): + raise TypeError("options must be ApplyOptions.") + + device = device.model_copy(deep=True) + blueprint = self._load_blueprint(blueprint).model_copy(deep=True) + registry = self._registry.copy() + effective_options = options or self._options + resolved = resolve_blueprint( + blueprint, + scope=scope, + inventory_variant=inventory_variant, + topology_variant=topology_variant, + registry=registry, + overrides=device.inventory_overrides, + ) + captured = _Captured( + device=device, + resolved=resolved, + naming=NamingScheme.layered(DEFAULT_NAMING, self._naming, resolved.naming, naming), + options=effective_options, + registry=registry, + ) + return _Planner(self._app, captured, scope, self._executor()).build() + + def apply( + self, + device: BlueprintDevice, + blueprint: Blueprint | str | PathLike[str], + *, + scope: Scope = "all", + inventory_variant: str = "default", + topology_variant: str = "default", + naming: NamingScheme | None = None, + options: ApplyOptions | None = None, + dry_run: bool = False, + ) -> ApplyResult: + """Load/plan and apply; identical to ``self.plan(...).apply(dry_run=dry_run)``. + + Accepts the same Blueprint instance or YAML file path as :meth:`plan`. + """ + return self.plan( + device, + blueprint, + scope=scope, + inventory_variant=inventory_variant, + topology_variant=topology_variant, + naming=naming, + options=options, + ).apply(dry_run=dry_run) + + def __repr__(self) -> str: + return f"BlueprintEngine(processors={self._registry.ids()}, options={self._options!r})" + + # --- Internal --- + + @staticmethod + def _load_blueprint(blueprint: Blueprint | str | PathLike[str]) -> Blueprint: + if isinstance(blueprint, Blueprint): + return blueprint + if isinstance(blueprint, (str, PathLike)): + return Blueprint.load(blueprint) + raise TypeError("blueprint must be a Blueprint instance or a YAML file path (str or PathLike[str]).") + + def _executor(self) -> _Executor: + return _Executor(self._app, clock=self._clock, sleep=self._sleep, logger=self._logger) + + +# --- Internal: captured inputs --- + + +class _Captured(BaseModel): + """Copies of everything a plan needs to execute, including deferred phases.""" + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + device: BlueprintDevice + resolved: ResolvedBlueprint + naming: NamingScheme + options: ApplyOptions + registry: ProcessorRegistry + inventory_work: InventoryWork | None = None + topology_target: TopologyTarget | None = None + topology_work: TopologyWork | None = None + topology_deferred: bool = False + + @property + def source(self) -> SourceFacts: + device = self.device + return SourceFacts( + key=device.key, + label=device.label, + description=device.description, + module_position=device.module_position, + attributes=device.model_dump(mode="json")["attributes"], + ) + + +# --- Internal: planning --- + + +class _Planner: + def __init__(self, app: BlueprintApp, captured: _Captured, scope: Scope, executor: _Executor) -> None: + self._app = app + self._captured = captured + self._scope = scope + self._executor = executor + self._inventory_gateway: InventoryGateway | None = None + + def build(self) -> BlueprintPlan: + captured = self._captured + resolved = captured.resolved + phases: list[PlannedPhase] = [] + diagnostics: list[Diagnostic] = [] + + inventory_work, current = self._plan_inventory(phases) + target, deferred_reason, topology_work = self._plan_topology(inventory_work, current, phases, diagnostics) + + captured = captured.model_copy( + update={ + "inventory_work": inventory_work, + "topology_target": target, + "topology_work": topology_work, + "topology_deferred": deferred_reason is not None, + } + ) + plan = BlueprintPlan( + source_key=captured.device.key, + scope=self._scope, + inventory_variant=resolved.inventory.variant if resolved.inventory else None, + topology_variant=resolved.topology.variant if resolved.topology else None, + blueprint_digest=resolved.digest, + driver_id=resolved.inventory.driver_id if resolved.inventory else None, + driver_schema_version=resolved.inventory.schema_version if resolved.inventory else None, + processor_type=resolved.topology.processor_id if resolved.topology else None, + inventory_id=captured.device.inventory_id, + topology_target=target, + phases=phases, + interface_bindings=list(topology_work.bindings) if topology_work else [], + diagnostics=diagnostics + (list(topology_work.diagnostics) if topology_work else []), + skipped_sections=dict(resolved.skipped), + fully_resolved=deferred_reason is None, + ) + plan._captured = captured + plan._executor = self._executor + return plan + + def _plan_inventory(self, phases: list[PlannedPhase]) -> tuple[InventoryWork | None, InventoryDevice | None]: + resolved = self._captured.resolved + if resolved.inventory is None: + phases.append(PlannedPhase(name="inventory", status="skipped", reason=resolved.skipped.get("inventory"))) + return None, None + naming = self._captured.naming + context = name_context(self._captured.source) + label = render_name("inventory_label", naming.inventory_label, context) if naming.inventory_label else None + description = ( + render_name("inventory_description", naming.inventory_description, context) + if naming.inventory_description + else None + ) + work, current = plan_inventory( + self._inventory(), resolved.inventory, self._captured.device, label=label, description=description + ) + operation = work.operation + phases.append( + PlannedPhase( + name="inventory", + status="planned" if operation else "no_change", + operations=[operation] if operation else [], + ) + ) + return work, current + + def _plan_topology( + self, + inventory_work: InventoryWork | None, + current: InventoryDevice | None, + phases: list[PlannedPhase], + diagnostics: list[Diagnostic], + ) -> tuple[TopologyTarget | None, str | None, TopologyWork | None]: + captured = self._captured + resolved = captured.resolved + if resolved.topology is None: + reason = resolved.skipped.get("topology") + phases.extend(PlannedPhase(name=name, status="skipped", reason=reason) for name in _TOPOLOGY_PHASES) + return None, None, None + + device = captured.device + target: TopologyTarget | None = device.topology + if target is None and device.inventory_id is not None: + target = DeviceTarget(device_id=device.inventory_id) + creating = inventory_work is not None and inventory_work.action == "create" + if target is None and not creating: + raise BlueprintTargetError( + f"Topology configuration for '{device.key}' needs an explicit 'topology' target or an 'inventory_id'." + ) + + reason: str | None = None + if target is None: + reason = "the topology device is the Inventory record created by this plan" + elif inventory_work is not None and inventory_work.affects_topology: + reason = "planned Inventory changes may alter the discovered topology" + else: + reason = self._membership_reason(target, diagnostics) + + if reason is not None: + phases.extend(PlannedPhase(name=name, status="deferred", reason=reason) for name in _TOPOLOGY_PHASES) + return target, reason, None + + assert target is not None + work = self._materialize(target, current) + phases.append(PlannedPhase(name="discovery", status="skipped", reason="topology already present")) + phases.append(PlannedPhase(name="topology_sync", status="no_change")) + phases.append( + PlannedPhase( + name="topology", + status="planned" if work.has_topology_writes else "no_change", + operations=work.topology_operations, + ) + ) + phases.append( + PlannedPhase( + name="module_tags", + status="planned" if work.has_module_writes else ("no_change" if work.module_id else "skipped"), + reason=None if work.module_id else "device target", + operations=work.module_operations, + ) + ) + return target, None, work + + def _membership_reason(self, target: TopologyTarget, diagnostics: list[Diagnostic]) -> str | None: + sync = self._captured.options.sync + gateway = InspectGateway(self._app.inspect) + if not gateway.in_topology(target.device_id): + if sync == "none" or isinstance(target, ModuleTarget): + raise BlueprintTargetError( + f"Inspect device '{target.device_id}' is not in the topology" + + ( + " (sync='none')." + if sync == "none" + else "; a module target requires its device in the topology." + ) + ) + return "the device must first be added to the topology" + info = gateway.sync_info(target.device_id) + if not _sync_pending(info): + return None + if sync == "none" or isinstance(target, ModuleTarget): + # A module target never synchronizes its whole parent device implicitly. + diagnostics.append( + Diagnostic( + level="warning", + code="topology.sync_pending", + message="Synchronization is pending but not performed (sync='none' or a module target); " + "planned against the current topology.", + entity_id=target.device_id, + ) + ) + return None + if sync == "add_only" and (info.update or info.remove): + raise BlueprintCapabilityError( + f"Synchronizing '{target.device_id}' requires updates/removals; sync='add_only' is insufficient " + "(use sync='reconcile' deliberately, or synchronize through the Inspect API)." + ) + return "topology synchronization is pending" + + def _materialize(self, target: TopologyTarget, current: InventoryDevice | None) -> TopologyWork: + return _materialize_topology(self._app, self._captured, target, self._inventory, current) + + def _inventory(self) -> InventoryGateway: + if self._inventory_gateway is None: + self._inventory_gateway = InventoryGateway(self._app.inventory) + return self._inventory_gateway + + +def _materialize_topology( + app: BlueprintApp, + captured: _Captured, + target: TopologyTarget, + inventory: Callable[[], InventoryGateway], + current: InventoryDevice | None, +) -> TopologyWork: + """Collect a fresh scoped context and compute exact topology edits from the captured configuration.""" + assert captured.resolved.topology is not None + scope = InspectGateway(app.inspect).read_scope(target) + own = _driver_context(current) if current is not None else None + owner = ( + own if own is not None and own.inventory_id == target.device_id else _owner_context(inventory, target.device_id) + ) + return compute_topology_work( + scope=scope, + resolved=captured.resolved.topology, + naming=captured.naming, + source=captured.source, + owner=owner, + inventory=own, + allow_label_collisions=captured.options.naming_collisions == "allow", + ) + + +# --- Internal: execution --- + + +class _Executor: + """Executes plans through the bound app; uses only the plan's captured settings.""" + + def __init__( + self, app: BlueprintApp, *, clock: Callable[[], float], sleep: Callable[[float], None], logger: logging.Logger + ) -> None: + self._app = app + self._clock = clock + self._sleep = sleep + self._logger = logger + + def execute(self, plan: BlueprintPlan, *, dry_run: bool = False) -> ApplyResult: + return _Execution(self, plan, dry_run=dry_run).run() + + +class _Execution: + def __init__(self, executor: _Executor, plan: BlueprintPlan, *, dry_run: bool) -> None: + self._executor = executor + self._app = executor._app + self._plan = plan + self._captured: _Captured = plan._captured + self._dry_run = dry_run + self._would_write = False + self._result = ApplyResult(source_key=plan.source_key, inventory_id=plan.inventory_id, dry_run=dry_run) + self._phases: dict[str, PhaseResult] = { + phase.name: PhaseResult( + name=phase.name, + status=phase.status if phase.status in ("skipped", "no_change") else "not_run", + message=phase.reason if phase.status == "skipped" else None, + ) + for phase in plan.phases + } + self._phases["verification"] = PhaseResult(name="verification", status="not_run") + self._current: str | None = None + self._unknown = False + self._wrote: list[str] = [] + self._mismatches: list[str] = [] + self._verification_errors: list[str] = [] + self._inventory_gateway: InventoryGateway | None = None + self._current_inventory: InventoryDevice | None = None + self._topology_readback_done = False + + def run(self) -> ApplyResult: + try: + self._run_inventory() + if self._captured.resolved.topology is not None: + self._run_topology() + except Exception as exc: # noqa: BLE001 + self._fail(exc) + self._finish() + return self._result + + # --- Phases --- + + def _run_inventory(self) -> None: + work = self._captured.inventory_work + if work is None: + return + phase = self._enter("inventory") + gateway = self._inventory() + if work.action == "none": + phase.status = "no_change" + return + if work.action == "create": + recheck_conflicts(gateway, self._captured.device, work) + candidate = build_candidate(new_device(work.driver_id), work.desired) + if self._dry_run: + self._planned(phase, [work.operation]) + return + with self._write(create=True): + online = gateway.create(candidate) + self._result.inventory_id = online.device_id + else: + assert work.inventory_id is not None + fresh = gateway.read(work.inventory_id) + check_baseline(work, fresh) + if self._dry_run: + self._planned(phase, [work.operation]) + return + with self._write(): + online = gateway.update(build_candidate(fresh, work.desired)) + self._current_inventory = online + self._mismatches.extend(f"inventory.{field}" for field in verify(online, work.desired)) + phase.status = "completed" + phase.operations = [work.operation] if work.operation else [] + phase.message = f"Inventory record '{online.device_id}'" + self._wrote.append("inventory") + + def _run_topology(self) -> None: + captured = self._captured + if captured.topology_deferred and self._dry_run: + reason = self._plan.phase("topology").reason if self._plan.phase("topology") else None + for name in _TOPOLOGY_PHASES: + phase = self._enter(name) + phase.status = "deferred" + phase.message = f"not run in a dry run: {reason}" + self._would_write = True + return + target = captured.topology_target + if target is None: + if self._result.inventory_id is None: + raise BlueprintTargetError("No topology target: the Inventory record id is unknown.") + target = DeviceTarget(device_id=self._result.inventory_id) + self._result.topology_device_id = target.device_id + self._result.module_id = target.module_id if isinstance(target, ModuleTarget) else None + gateway = InspectGateway(self._app.inspect) + + if captured.topology_deferred: + work = self._discover_and_materialize(gateway, target) + self._result.materialized = True + else: + assert captured.topology_work is not None + work = captured.topology_work + self._enter("topology") + if gateway.read_scope(target).fingerprint != work.fingerprint: + raise BlueprintConflictError( + f"The topology of {_target_label(target)} changed since planning; create a new plan." + ) + self._result.interface_bindings = list(work.bindings) + self._result.diagnostics.extend(work.diagnostics) + + overlap = gateway.staged_edit_keys() & work.touched_keys() + if overlap: + entities = ", ".join(f"{kind} '{entity_id}'" for kind, entity_id in sorted(overlap)) + raise BlueprintConflictError( + f"Pending uncommitted Inspect edits overlap this plan ({entities}); commit or discard them first." + ) + + phase = self._enter("topology") + if work.has_topology_writes and self._dry_run: + self._planned(phase, work.topology_operations) + elif work.has_topology_writes: + with self._write(): + gateway.commit(work) + phase.status = "completed" + phase.operations = list(work.topology_operations) + self._wrote.append("topology") + else: + phase.status = "no_change" + + self._run_module_tags(gateway, work) + if self._wrote and ("topology" in self._wrote or "module_tags" in self._wrote): + try: + self._mismatches.extend(verify_topology(gateway.read_scope(target), work)) + except Exception as exc: # noqa: BLE001 - verification never turns an applied write into a failure + self._verification_errors.append(f"topology read-back failed: {type(exc).__name__}: {exc}") + finally: + self._topology_readback_done = True + + def _run_module_tags(self, gateway: InspectGateway, work: TopologyWork) -> None: + phase = self._enter("module_tags") + if work.module_id is None: + phase.status = "skipped" + phase.message = "device target" + return + if not work.has_module_writes: + phase.status = "no_change" + return + current = gateway.module_local_tags(work.device_id, work.module_id) + if work.module_tag_baseline is not None and set(current or ()) != set(work.module_tag_baseline): + raise BlueprintConflictError( + f"Local tags of module '{work.module_id}' changed since planning; the tag phase was not applied." + ) + if self._dry_run: + self._planned(phase, work.module_operations) + return + for operation in work.module_operations: + tag = operation.changes[0].after + with self._write(): + if operation.action == "assign_tag": + gateway.assign_tag(tag, work.module_id) + else: + gateway.unassign_tag(tag, work.module_id) + phase.operations.append(operation) + if "module_tags" not in self._wrote: + self._wrote.append("module_tags") + gateway.refresh_device(work.device_id) + phase.status = "completed" + + def _discover_and_materialize(self, gateway: InspectGateway, target: TopologyTarget) -> TopologyWork: + options = self._captured.options + discovery = self._enter("discovery") + deadline = self._executor._clock() + options.discovery_timeout + sync_phase = self._phases["topology_sync"] + sync_phase.status = "no_change" + while True: + try: + self._ensure_membership(gateway, target) + self._enter("topology") + work = _materialize_topology( + self._app, self._captured, target, self._inventory, self._current_inventory + ) + discovery.status = "completed" + return work + except (ProcessorInputError, BlueprintTargetError) as exc: + if self._unknown or isinstance(exc, _NoRetry): + raise + remaining = deadline - self._executor._clock() + if remaining <= 0: + self._current = "discovery" + raise type(exc)(f"Topology not ready after {options.discovery_timeout:g}s: {exc}") from exc + self._executor._sleep(min(options.poll_interval, remaining)) + + def _ensure_membership(self, gateway: InspectGateway, target: TopologyTarget) -> None: + sync = self._captured.options.sync + phase = self._enter("topology_sync") + if isinstance(target, ModuleTarget): + # Never add or synchronize a whole parent device to reach a module target. + if not gateway.in_topology(target.device_id): + raise _NoRetry( + f"Parent device '{target.device_id}' of module '{target.module_id}' is not in the topology." + ) + return + if not gateway.in_topology(target.device_id): + if sync == "none": + raise _NoRetry(f"Inspect device '{target.device_id}' is not in the topology (sync='none').") + with self._write(): + gateway.add_to_topology(target.device_id) + self._record_sync(phase, "add_to_topology", target.device_id) + info = gateway.sync_info(target.device_id) + if not _sync_pending(info) or sync == "none": + return + if sync == "add_only" and (info.update or info.remove): + raise BlueprintCapabilityError( + f"Synchronizing '{target.device_id}' requires updates/removals; sync='add_only' is insufficient." + ) + with self._write(): + gateway.sync(target.device_id, add_only=sync == "add_only") + self._record_sync(phase, "sync", target.device_id) + + def _record_sync(self, phase: PhaseResult, action: str, device_id: str) -> None: + phase.operations.append(PlannedOperation(action=action, entity_kind="device", entity_id=device_id)) # type: ignore[arg-type] + phase.status = "completed" + if "topology_sync" not in self._wrote: + self._wrote.append("topology_sync") + + # --- Bookkeeping --- + + def _planned(self, phase: PhaseResult, operations: list[PlannedOperation | None]) -> None: + """Dry run: record the writes that would run instead of performing them.""" + phase.status = "planned" + phase.operations = [operation for operation in operations if operation is not None] + phase.message = "dry run: not applied" + self._would_write = True + + def _enter(self, name: str) -> PhaseResult: + self._current = name + phase = self._phases.setdefault(name, PhaseResult(name=name, status="not_run")) # type: ignore[arg-type] + return phase + + @contextlib.contextmanager + def _write(self, *, create: bool = False) -> Iterator[None]: + """Mark a server write; an unexpected exception means the outcome is unknown.""" + try: + yield + except Exception as exc: + self._unknown = not _known_rejection(exc, create=create) + raise + + def _fail(self, exc: Exception) -> None: + name = self._current or "inventory" + phase = self._phases.setdefault(name, PhaseResult(name=name, status="not_run")) # type: ignore[arg-type] + phase.status = "unknown" if self._unknown else "failed" + phase.message = f"{type(exc).__name__}: {exc}" + if self._unknown: + self._result.status = "unknown" + elif self._wrote: + self._result.status = "partial" + else: + self._result.status = "failed" + self._error = exc + + def _finish(self) -> None: + result = self._result + if self._wrote: + verification = self._phases["verification"] + if ("topology" in self._wrote or "module_tags" in self._wrote) and not self._topology_readback_done: + self._verification_errors.append("topology read-back did not run") + if self._mismatches or self._verification_errors: + result.verification = "unconfirmed" + result.verification_detail = "; ".join( + ([f"read-back differs for {', '.join(self._mismatches)}"] if self._mismatches else []) + + self._verification_errors + ) + else: + result.verification = "confirmed" + verification.status = "completed" + else: + self._phases["verification"].status = "skipped" + result.phases = [self._phases[name] for name in _PHASE_ORDER if name in self._phases] + error = getattr(self, "_error", None) + if error is not None: + self._executor._logger.warning( + "Blueprint apply for '%s' ended %s: %s", result.source_key, result.status, error + ) + raise BlueprintApplyError(result, f"{type(error).__name__}: {error}") from error + if self._dry_run: + result.verification_detail = "dry run: nothing was written" + result.status = "planned" if self._would_write else "no_change" + else: + result.status = "succeeded" if self._wrote else "no_change" + self._executor._logger.info("Blueprint apply for '%s' %s.", result.source_key, result.status) + + def _inventory(self) -> InventoryGateway: + if self._inventory_gateway is None: + self._inventory_gateway = InventoryGateway(self._app.inventory) + return self._inventory_gateway + + +# --- Internal helpers --- + +_TOPOLOGY_PHASES: tuple[PhaseName, ...] = ("discovery", "topology_sync", "topology", "module_tags") +_PHASE_ORDER: tuple[str, ...] = ("inventory", "discovery", "topology_sync", "topology", "module_tags", "verification") + + +class _NoRetry(BlueprintTargetError): + """A target error that waiting cannot resolve.""" + + +def _known_rejection(exc: Exception, *, create: bool) -> bool: + """Whether a write error proves the server did not apply the change.""" + if isinstance(exc, (InspectCommitError, InspectCommitConflictError, BlueprintError)): + return True + if isinstance(exc, ValueError): + message = str(exc) + return ( + message.startswith("Failed to add device") + if create + else message.startswith( + ( + "Failed to update device", + "Failed to add device", + "Failed to retrieve existing device configuration", + ) + ) + ) + return False + + +def _sync_pending(info: Any) -> bool: + return info is not None and bool(info.add or info.update or info.remove) + + +def _target_label(target: TopologyTarget) -> str: + if isinstance(target, ModuleTarget): + return f"module '{target.module_id}' of device '{target.device_id}'" + return f"device '{target.device_id}'" + + +def _driver_context(device: InventoryDevice) -> DriverContext: + return DriverContext( + inventory_id=device.device_id, + driver_id=device.driver_id, + address=device.configuration.config.cinfo.address or None, + custom_settings=device.configuration.config.customSettings.model_dump(mode="json", exclude={"driver_id"}), + ) + + +def _owner_context(inventory: Callable[[], InventoryGateway], device_id: str) -> DriverContext | None: + try: + validate_device_id(device_id) + except ValueError: + return None # e.g. virtual devices have no Inventory record + try: + return _driver_context(inventory().read(device_id)) + except (BlueprintTargetError, BlueprintValidationError): + return None + + +BlueprintPlan.model_rebuild() + +__all__ = ["BlueprintApp", "BlueprintEngine", "BlueprintPlan"] diff --git a/src/videoipath_automation_tool/blueprints/errors.py b/src/videoipath_automation_tool/blueprints/errors.py new file mode 100644 index 0000000..6cdaac3 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/errors.py @@ -0,0 +1,103 @@ +"""Typed exceptions for blueprint loading, planning, and application. + +Every error derives from :class:`BlueprintError`. Validation problems carry structured +:class:`ValidationIssue` records (document path, optional YAML location, issue code) instead of +raw values, so messages never echo credentials or arbitrary server payloads. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING + +from pydantic import BaseModel, ConfigDict + +if TYPE_CHECKING: + from videoipath_automation_tool.blueprints.models import ApplyResult + + +class ValidationIssue(BaseModel): + """One validation finding. ``path`` is the dotted document path (e.g. + ``topology.receiver.vertex_processor.params.redundant_streams``).""" + + model_config = ConfigDict(frozen=True) + + path: str = "" + message: str + code: str = "invalid" + source: str | None = None + line: int | None = None + column: int | None = None + + def __str__(self) -> str: + lines: list[str] = [] + if self.source or self.line is not None: + location = self.source or "" + if self.line is not None: + location += f":{self.line}:{self.column or 1}" + lines.append(location) + if self.path: + lines.append(self.path) + lines.append(self.message) + return "\n".join(lines) + + +class BlueprintError(Exception): + """Base class for all blueprint errors.""" + + +class BlueprintValidationError(BlueprintError): + """A blueprint, device description, naming rule, or processor parameter set is invalid.""" + + def __init__(self, issues: Iterable[ValidationIssue] | ValidationIssue | str, *, code: str = "invalid") -> None: + if isinstance(issues, str): + issues = [ValidationIssue(message=issues, code=code)] + elif isinstance(issues, ValidationIssue): + issues = [issues] + self.issues: list[ValidationIssue] = list(issues) + super().__init__("\n\n".join(str(issue) for issue in self.issues) or "Blueprint validation failed.") + + @property + def codes(self) -> list[str]: + return [issue.code for issue in self.issues] + + +class BlueprintTargetError(BlueprintError): + """A binding is missing or ambiguous, or a target lies outside the requested scope.""" + + +class ProcessorInputError(BlueprintError): + """A processor cannot interpret the observed topology (unsupported or incomplete layout).""" + + +class BlueprintCapabilityError(BlueprintError): + """A requested operation is not supported by the adapter or the server.""" + + +class BlueprintConflictError(BlueprintError): + """A reviewed plan or a managed baseline changed since it was captured; replan.""" + + +class BlueprintApplyError(BlueprintError): + """Execution failed, partially succeeded, or ended with an unknown outcome. + + ``result`` is the structured :class:`ApplyResult` describing every completed, failed, and + unknown phase, including any Inventory id created before the failure. The original error is + chained as ``__cause__``. + """ + + def __init__(self, result: ApplyResult, message: str) -> None: + self.result = result + super().__init__(f"Blueprint apply {result.status} for '{result.source_key}': {message}") + + +__all__ = [ + "BlueprintApplyError", + "BlueprintCapabilityError", + "BlueprintConflictError", + "BlueprintError", + "BlueprintTargetError", + "BlueprintValidationError", + "ProcessorInputError", + "ValidationIssue", +] diff --git a/src/videoipath_automation_tool/blueprints/inspect.py b/src/videoipath_automation_tool/blueprints/inspect.py new file mode 100644 index 0000000..02784a4 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/inspect.py @@ -0,0 +1,883 @@ +"""Inspect integration: exact device/module scoping, fresh detached reads, interface bindings, +topology work computation, and the existing Inspect write paths. + +Planning never touches the live snapshot's domain objects: scope data is built from fresh collector +and lookup reads, so pending (uncommitted) user edits on the shared snapshot are neither read as +server truth nor flushed. Writes stage exactly the planned id/field intents in one +``InspectTransaction`` (with its conflict check); module tags use the separate ``assignTag`` / +``unassignTag`` actions and are tracked as their own phase. Module tag baselines come from the +module's *local* assignments — inherited/effective tags are never used as a removal baseline. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import defaultdict +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from videoipath_automation_tool.apps.inspect.model.collector import ( + InspectApiDoubleVertexInfo, + InspectApiNodeStatusItem, + InspectApiSingleVertexInfo, +) +from videoipath_automation_tool.apps.inspect.model.tags import module_resource_id +from videoipath_automation_tool.blueprints.errors import ( + BlueprintCapabilityError, + BlueprintError, + BlueprintTargetError, + BlueprintValidationError, + ProcessorInputError, + ValidationIssue, +) +from videoipath_automation_tool.blueprints.models import ( + Coordinates, + Diagnostic, + EndpointIdentity, + FieldChange, + InterfaceBinding, + ModuleTarget, + PlannedOperation, + ProcessorResult, + TagDelta, + TopologyTarget, +) +from videoipath_automation_tool.blueprints.naming import NameContext, NamingScheme, render_name +from videoipath_automation_tool.blueprints.processors import ( + DeviceRecord, + DriverContext, + ModuleRecord, + PortRecord, + ProcessingContext, + SourceFacts, + VertexRecord, +) +from videoipath_automation_tool.blueprints.resolution import ResolvedTopology + + +class ScopeData(BaseModel): + """Fresh, detached read of one device/module scope.""" + + model_config = ConfigDict(frozen=True) + + device_id: str + module_id: str | None = None + device: DeviceRecord + modules: tuple[ModuleRecord, ...] = () + ports: tuple[PortRecord, ...] = () + vertices: tuple[VertexRecord, ...] = () + external_endpoint_labels: dict[str, str] = Field(default_factory=dict) + """Labels of endpoint vertices of the same device outside the scope (collision checks).""" + without_form: frozenset[str] = frozenset() + + @property + def fingerprint(self) -> str: + document = self.model_dump( + mode="json", + include={ + "device_id", + "module_id", + "device", + "modules", + "ports", + "vertices", + "external_endpoint_labels", + }, + ) + return hashlib.sha256(json.dumps(document, sort_keys=True).encode()).hexdigest() + + def vertex(self, vertex_id: str) -> VertexRecord | None: + return next((vertex for vertex in self.vertices if vertex.id == vertex_id), None) + + def module(self) -> ModuleRecord | None: + return next((module for module in self.modules if module.id == self.module_id), None) + + +class TopologyWork(BaseModel): + """Exact topology edits for one scope (internal; unredacted values are never secrets here).""" + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + device_id: str + module_id: str | None = None + fingerprint: str + device_intents: dict[str, Any] = Field(default_factory=dict) + vertex_intents: dict[str, dict[str, Any]] = Field(default_factory=dict) + module_tag_baseline: tuple[str, ...] | None = None + tags_to_assign: list[str] = Field(default_factory=list) + tags_to_unassign: list[str] = Field(default_factory=list) + expected_device: dict[str, Any] = Field(default_factory=dict) + expected_vertices: dict[str, dict[str, Any]] = Field(default_factory=dict) + expected_module_tags: list[str] | None = None + bindings: list[InterfaceBinding] = Field(default_factory=list) + diagnostics: list[Diagnostic] = Field(default_factory=list) + topology_operations: list[PlannedOperation] = Field(default_factory=list) + module_operations: list[PlannedOperation] = Field(default_factory=list) + + @property + def has_topology_writes(self) -> bool: + return bool(self.device_intents or self.vertex_intents) + + @property + def has_module_writes(self) -> bool: + return bool(self.tags_to_assign or self.tags_to_unassign) + + def touched_keys(self) -> set[tuple[str, str]]: + keys = {("vertex", vertex_id) for vertex_id in self.vertex_intents} + if self.device_intents: + keys.add(("device", self.device_id)) + if self.has_module_writes and self.module_id is not None: + keys.add(("module", self.module_id)) + return keys + + +class InspectGateway: + """The only place that talks to ``app.inspect`` (public methods plus one internal API handle).""" + + def __init__(self, inspect_app: Any) -> None: + self._app = inspect_app + + @property + def _api(self) -> Any: + return self._app._inspect_api + + # --- Reads --- + + def in_topology(self, device_id: str) -> bool: + if self._detail(device_id) is not None: + return True + return device_id.startswith("virtual.") and self._device_form(device_id) is not None + + def sync_info(self, device_id: str) -> Any | None: + """Pending sync for ``device_id``, or ``None`` when the server reports no sync record. + + A failed lookup is an error. Treating it as "nothing pending" would let apply commit topology + edits while a synchronization that updates or removes elements is still outstanding. + """ + try: + return self._app.get_sync_info([device_id]).get(device_id) + except BlueprintError: + raise + except Exception as exc: + raise BlueprintError( + f"Could not read synchronization status for '{device_id}': {type(exc).__name__}: {exc}" + ) from exc + + def read_scope(self, target: TopologyTarget) -> ScopeData: + device_id = target.device_id + module_id = target.module_id if isinstance(target, ModuleTarget) else None + node = self._detail(device_id) + form = self._device_form(device_id) + if node is None and form is None: + raise BlueprintTargetError(f"Inspect device '{device_id}' is not in the topology.") + + modules = _modules_of(node) + if module_id is not None and module_id not in modules: + known = ", ".join(sorted(modules)[:10]) or "none" + raise BlueprintTargetError( + f"Module '{module_id}' does not belong to Inspect device '{device_id}' (modules: {known})." + ) + + ports: list[tuple[PortRecord, list[tuple[str, InspectApiSingleVertexInfo]]]] = [] + for mod_id, module in modules.items(): + for key, port in _items(module.ports): + port_id = port.pid or port.id or key + sides = _vertex_sides(port.parsed_vertex_info) + record = PortRecord( + id=port_id, + module_id=mod_id, + factory_label=port.label, + label=port.effective_label, + vertex_ids=tuple(vid for vid, _ in sides), + ) + ports.append((record, sides)) + + in_scope = [(p, sides) for p, sides in ports if module_id is None or p.module_id == module_id] + scope_ids = [vid for _, sides in in_scope for vid, _ in sides] + external_ids = [ + vid + for p, sides in ports + if module_id is not None and p.module_id != module_id + for vid, side in sides + if side.fields is not None and side.fields.isEndpoint + ] + forms = self._vertex_forms(scope_ids + external_ids) + + vertices = sorted( + (_vertex_record(vid, side, port, forms.get(vid)) for port, sides in in_scope for vid, side in sides), + key=lambda record: record.id, + ) + external = {vid: forms[vid].fields.label for vid in external_ids if vid in forms and forms[vid].fields.label} + module_records = tuple( + _module_record(mod_id, module) for mod_id, module in sorted(modules.items()) if module_id in (None, mod_id) + ) + return ScopeData( + device_id=device_id, + module_id=module_id, + device=_device_record(device_id, node, form), + modules=module_records, + ports=tuple(sorted((p for p, _ in in_scope), key=lambda record: record.id)), + vertices=tuple(vertices), + external_endpoint_labels=external, + without_form=frozenset(vid for vid in scope_ids if vid not in forms), + ) + + def module_local_tags(self, device_id: str, module_id: str) -> tuple[str, ...] | None: + node = self._detail(device_id) + module = _modules_of(node).get(module_id) + if module is None: + raise BlueprintTargetError(f"Module '{module_id}' of '{device_id}' is no longer in the topology.") + return _local_tags(module) + + def staged_edit_keys(self) -> set[tuple[str, str]]: + """Entities with pending (uncommitted) domain-object edits on the shared app snapshot.""" + snapshot = getattr(self._app, "_snapshot", None) + if snapshot is None: + return set() + return {(kind, entity_id) for kind, entity_id, intents in snapshot.iter_staged_edits() if intents} + + # --- Writes --- + + def add_to_topology(self, device_id: str) -> None: + if not self._app.add_devices_to_topology([device_id], sync=False): + raise BlueprintTargetError(f"addDevices reported failure for '{device_id}' (not discovered yet?).") + + def sync(self, device_id: str, *, add_only: bool) -> None: + if not self._app.sync_devices([device_id], add_only=add_only): + raise BlueprintTargetError(f"syncDevices reported failure for '{device_id}'.") + + def commit(self, work: TopologyWork) -> Any: + with self._app.transaction() as tx: + if work.device_intents: + tx.update_device(work.device_id, intents=dict(work.device_intents)) + for vertex_id, intents in sorted(work.vertex_intents.items()): + tx.update_vertex(vertex_id, intents=dict(intents)) + return tx.commit() + + def assign_tag(self, tag_id: str, module_id: str) -> None: + _check_action(self._api.assign_tag(tag_id, [module_resource_id(module_id)]), "assignTag", tag_id) + + def unassign_tag(self, tag_id: str, module_id: str) -> None: + _check_action(self._api.unassign_tag(tag_id, [module_resource_id(module_id)]), "unassignTag", tag_id) + + def refresh_device(self, device_id: str) -> None: + snapshot = getattr(self._app, "_snapshot", None) + if snapshot is not None: + snapshot.apply_post_commit(device_ids=[device_id]) + + # --- Internal --- + + def _detail(self, device_id: str) -> InspectApiNodeStatusItem | None: + node = self._api.get_device_detail(device_id) + if node is None and device_id.startswith("virtual."): + node = self._api.get_device_detail(device_id.replace(".", "-")) + return node + + def _device_form(self, device_id: str) -> Any | None: + try: + return self._api.lookup_inspect_device(device_id).data.fields + except Exception: # noqa: BLE001 + return None + + def _vertex_forms(self, vertex_ids: list[str]) -> dict[str, Any]: + if not vertex_ids: + return {} + return dict(self._api.lookup_vertices(vertex_ids).data) + + +def resolve_interfaces( + mapping: dict[str, list[str]] | None, scope: ScopeData, module_position: str | None +) -> list[InterfaceBinding]: + """Resolve caller interface keys to exact ports (with IP vertices) in scope, in fallback order.""" + if not mapping: + return [] + by_id = {vertex.id: vertex for vertex in scope.vertices} + bindings: list[InterfaceBinding] = [] + for key, candidates in mapping.items(): + attempted: list[str] = [] + for raw in candidates: + label = _substitute_position(raw, module_position, key) + attempted.append(label) + ports = [ + port + for port in scope.ports + if port.factory_label == label and any(by_id[v].kind == "ip" for v in port.vertex_ids if v in by_id) + ] + if len(ports) > 1: + raise BlueprintTargetError( + f"Interface '{key}': factory label '{label}' matches {len(ports)} ports " + f"({', '.join(p.id for p in ports)}) in {_scope_label(scope)}." + ) + if ports: + port = ports[0] + vertices = [by_id[v] for v in port.vertex_ids if v in by_id] + bindings.append( + InterfaceBinding( + key=key, + candidate=label, + port_id=port.id, + in_vertex_id=next((v.id for v in vertices if v.vertex_type == "In"), None), + out_vertex_id=next((v.id for v in vertices if v.vertex_type == "Out"), None), + ) + ) + break + else: + raise BlueprintTargetError( + f"Interface '{key}': no IP port in {_scope_label(scope)} matches {', '.join(map(repr, attempted))}." + ) + return bindings + + +def build_context( + scope: ScopeData, + source: SourceFacts, + bindings: list[InterfaceBinding], + owner: DriverContext | None, + inventory: DriverContext | None, +) -> ProcessingContext: + return ProcessingContext( + source=source, + scope="module" if scope.module_id else "device", + device_id=scope.device_id, + module_id=scope.module_id, + device=scope.device, + modules=scope.modules, + ports=scope.ports, + vertices=scope.vertices, + interfaces=tuple(bindings), + owner=owner, + inventory=inventory + if inventory is None or owner is None or inventory.inventory_id != owner.inventory_id + else None, + capabilities=frozenset(), + ) + + +def compute_topology_work( + *, + scope: ScopeData, + resolved: ResolvedTopology, + naming: NamingScheme, + source: SourceFacts, + owner: DriverContext | None, + inventory: DriverContext | None, + allow_label_collisions: bool, +) -> TopologyWork: + """Run the processor on the detached scope and turn proposals into exact, compared edits.""" + config = resolved.config + bindings = resolve_interfaces(config.ip_vertex_mapping, scope, source.module_position) + context = build_context(scope, source, bindings, owner, inventory) + result = _run_processor(resolved, context) + is_module = scope.module_id is not None + processor_source = f"processor:{resolved.processor_id}" + + if is_module and result.device is not None and result.device.managed(): + raise BlueprintTargetError( + f"Processor '{resolved.processor_id}' proposed device-wide changes for a module target." + ) + if not is_module and result.module is not None and result.module.managed(): + raise BlueprintTargetError(f"Processor '{resolved.processor_id}' proposed module changes for a device target.") + if is_module and config.device is not None and config.device.managed(): + raise BlueprintTargetError( + "'topology.device' settings cannot be applied to a module target; use a device target." + ) + if not is_module and config.module is not None and config.module.managed(): + raise BlueprintTargetError("'topology.module' settings require a module target.") + + # Vertex proposals: processor < naming < explicit overrides. + vertex_desired: dict[str, dict[str, tuple[Any, str]]] = defaultdict(dict) + endpoints: dict[str, EndpointIdentity] = {} + for edit in result.vertices: + if scope.vertex(edit.vertex_id) is None: + raise BlueprintTargetError( + f"Processor '{resolved.processor_id}' proposed vertex '{edit.vertex_id}' outside {_scope_label(scope)}." + ) + for field, value in edit.fields.managed().items(): + existing = vertex_desired[edit.vertex_id].get(field) + if existing is not None and existing[0] != value: + raise ProcessorInputError( + f"Processor '{resolved.processor_id}' proposed conflicting '{field}' values for '{edit.vertex_id}'." + ) + vertex_desired[edit.vertex_id][field] = (value, processor_source) + if edit.endpoint is not None: + if edit.vertex_id in endpoints and endpoints[edit.vertex_id] != edit.endpoint: + raise ProcessorInputError( + f"Processor '{resolved.processor_id}' proposed conflicting endpoint identities for '{edit.vertex_id}'." + ) + endpoints[edit.vertex_id] = edit.endpoint + + for vertex_id, endpoint in endpoints.items(): + record = scope.vertex(vertex_id) + assert record is not None + facts = name_context(source, endpoint=endpoint, vertex=record) + for entry, field in (("endpoint_label", "label"), ("endpoint_description", "description")): + expression = getattr(naming, entry) + if expression is not None: + vertex_desired[vertex_id][field] = (render_name(entry, expression, facts), "naming") + + for index, override in enumerate(config.vertices or []): + vertex_id = _select_override(override, scope, source.module_position, index) + override_source = resolved.provenance.get("vertices", "blueprint") + for field, value in override.fields.managed().items(): + vertex_desired[vertex_id][field] = (value, f"override:{override_source}") + + for vertex_id, fields in vertex_desired.items(): + record = scope.vertex(vertex_id) + assert record is not None + if vertex_id in scope.without_form: + raise BlueprintTargetError(f"Vertex '{vertex_id}' has no edit form on the server; it cannot be configured.") + for field, kind in _KIND_SPECIFIC_FIELDS.items(): + if field in fields and record.kind != kind: + raise BlueprintCapabilityError( + f"'{field}' is only supported on {kind} vertices ('{vertex_id}' is '{record.kind}')." + ) + + # Device and module proposals: processor defaults < explicit blueprint < naming (device label). + device_desired: dict[str, tuple[Any, str]] = {} + module_tags: tuple[Any, str] | None = None + if not is_module: + if result.device is not None: + device_desired.update({k: (v, processor_source) for k, v in result.device.managed().items()}) + if config.device is not None: + device_desired.update( + { + k: (v, resolved.provenance.get(f"device.{k}", "blueprint")) + for k, v in config.device.managed().items() + } + ) + facts = name_context(source) + for entry, field in (("device_label", "label"), ("device_description", "description")): + expression = getattr(naming, entry) + if expression is not None: + device_desired[field] = (render_name(entry, expression, facts), "naming") + else: + if result.module is not None and result.module.tags is not None: + module_tags = (result.module.tags, processor_source) + if config.module is not None and config.module.tags is not None: + module_tags = (config.module.tags, resolved.provenance.get("module.tags", "blueprint")) + + work = _compare(scope, vertex_desired, endpoints, device_desired, module_tags) + _check_label_collisions(scope, work, allow=allow_label_collisions) + return work.model_copy( + update={"bindings": bindings, "diagnostics": list(result.diagnostics), "fingerprint": scope.fingerprint} + ) + + +def verify_topology(scope: ScopeData, work: TopologyWork) -> list[str]: + """Managed fields whose fresh read-back differs from the desired value.""" + mismatches: list[str] = [] + for field, value in work.expected_device.items(): + if not _same(field, _device_value(scope.device, field), value): + mismatches.append(f"device.{field}") + for vertex_id, fields in work.expected_vertices.items(): + record = scope.vertex(vertex_id) + for field, value in fields.items(): + if record is None or not _same(field, getattr(record, field), value): + mismatches.append(f"{vertex_id}.{field}") + if work.expected_module_tags is not None: + module = scope.module() + if module is None or module.local_tags is None or set(module.local_tags) != set(work.expected_module_tags): + mismatches.append(f"{work.module_id}.tags") + return mismatches + + +def name_context( + source: SourceFacts, *, endpoint: EndpointIdentity | None = None, vertex: VertexRecord | None = None +) -> NameContext: + """Naming facts for a source entity (and optionally one endpoint vertex).""" + return NameContext( + device={"key": source.key, "label": source.label, "description": source.description}, + module={"position": source.module_position}, + endpoint=endpoint.model_dump() if endpoint is not None else {}, + vertex={"factory_label": vertex.factory_label, "id": vertex.id} if vertex is not None else {}, + attributes=source.attributes, + ) + + +# --- Internal --- + +_DEVICE_WIRE = { + "label": "descriptor.label", + "description": "descriptor.desc", + "icon_type": "iconType", + "icon_size": "iconSize", + "sdp_strategy": "sdpStrategy", + "site_id": "siteId", + "coordinates": "coordinates", + "tags": "localAssignedTags", +} +_VERTEX_WIRE = { + "use_as_endpoint": "useAsEndpoint", + "label": "label", + "description": "desc", + "active": "active", + "sips_mode": "sipsMode", + "tags": "localAssignedTags", + "sdp_support": "typeFields.specific.sdpSupport", + "main_destination_port": "typeFields.generic.mainDstInfo.port", + "spare_destination_port": "typeFields.generic.spareDstInfo.port", + "supports_static_igmp": "typeFields.supportsStaticIgmpCfg", +} +_KIND_SPECIFIC_FIELDS = { + "sdp_support": "codec", + "main_destination_port": "codec", + "spare_destination_port": "codec", + "supports_static_igmp": "ip", +} +_POSITION_TOKEN = re.compile(r"\{([^{}]*)\}") + + +def _run_processor(resolved: ResolvedTopology, context: ProcessingContext) -> ProcessorResult: + if resolved.processor_cls is None: + return ProcessorResult() + try: + raw = resolved.processor_cls().process(context, resolved.params) + except BlueprintError: + raise + except Exception as exc: + raise ProcessorInputError(f"Processor '{resolved.processor_id}' failed: {type(exc).__name__}: {exc}") from exc + if not isinstance(raw, ProcessorResult): + raise ProcessorInputError(f"Processor '{resolved.processor_id}' must return a ProcessorResult.") + try: # re-validate even if a plugin bypassed normal construction + return ProcessorResult.model_validate(raw.model_dump(mode="python", exclude_unset=True, warnings=False)) + except Exception as exc: # noqa: BLE001 + raise ProcessorInputError(f"Processor '{resolved.processor_id}' returned an invalid result: {exc}") from None + + +def _compare( + scope: ScopeData, + vertex_desired: dict[str, dict[str, tuple[Any, str]]], + endpoints: dict[str, EndpointIdentity], + device_desired: dict[str, tuple[Any, str]], + module_tags: tuple[Any, str] | None, +) -> TopologyWork: + device_intents: dict[str, Any] = {} + expected_device: dict[str, Any] = {} + device_changes: list[FieldChange] = [] + for field, (value, source) in sorted(device_desired.items()): + current = _device_value(scope.device, field) + desired = _resolve_tags(value, current, f"device '{scope.device_id}'") if field == "tags" else _plain(value) + expected_device[field] = desired + if not _same(field, current, desired): + device_intents[_DEVICE_WIRE[field]] = _wire(field, desired) + device_changes.append(FieldChange(field=field, before=_json(current), after=_json(desired), source=source)) + + vertex_intents: dict[str, dict[str, Any]] = {} + expected_vertices: dict[str, dict[str, Any]] = {} + operations: list[PlannedOperation] = [] + if device_changes: + operations.append( + PlannedOperation(action="update", entity_kind="device", entity_id=scope.device_id, changes=device_changes) + ) + for vertex_id in sorted(vertex_desired): + record = scope.vertex(vertex_id) + assert record is not None + changes: list[FieldChange] = [] + for field, (value, source) in sorted(vertex_desired[vertex_id].items()): + current = getattr(record, field) + desired = _resolve_tags(value, current, f"vertex '{vertex_id}'") if field == "tags" else value + expected_vertices.setdefault(vertex_id, {})[field] = desired + if not _same(field, current, desired): + vertex_intents.setdefault(vertex_id, {})[_VERTEX_WIRE[field]] = _wire(field, desired) + changes.append(FieldChange(field=field, before=_json(current), after=_json(desired), source=source)) + if changes: + operations.append( + PlannedOperation( + action="update", + entity_kind="vertex", + entity_id=vertex_id, + changes=changes, + endpoint=endpoints.get(vertex_id), + ) + ) + + module_operations: list[PlannedOperation] = [] + baseline: tuple[str, ...] | None = None + assign: list[str] = [] + unassign: list[str] = [] + expected_tags: list[str] | None = None + if module_tags is not None and scope.module_id is not None: + spec, source = module_tags + module = scope.module() + baseline = module.local_tags if module is not None else None + if baseline is None: + if not isinstance(spec, TagDelta) or spec.remove: + raise BlueprintCapabilityError( + f"Local tags of module '{scope.module_id}' cannot be distinguished from inherited tags on this " + "server; only additive tag changes ({add: [...]}) are supported for it." + ) + assign = list(dict.fromkeys(spec.add)) + else: + desired = _resolve_tags(spec, baseline, f"module '{scope.module_id}'") + expected_tags = desired + assign = [tag for tag in desired if tag not in baseline] + unassign = [tag for tag in baseline if tag not in desired] + for action, tags in (("assign_tag", assign), ("unassign_tag", unassign)): + for tag in tags: + module_operations.append( + PlannedOperation( + action=action, # type: ignore[arg-type] + entity_kind="module", + entity_id=scope.module_id, + changes=[FieldChange(field="tags", after=tag, source=source)], + ) + ) + + return TopologyWork( + device_id=scope.device_id, + module_id=scope.module_id, + fingerprint=scope.fingerprint, + device_intents=device_intents, + vertex_intents=vertex_intents, + module_tag_baseline=baseline, + tags_to_assign=assign, + tags_to_unassign=unassign, + expected_device=expected_device, + expected_vertices=expected_vertices, + expected_module_tags=expected_tags, + topology_operations=operations, + module_operations=module_operations, + ) + + +def _check_label_collisions(scope: ScopeData, work: TopologyWork, *, allow: bool) -> None: + changed = {vid for vid, intents in work.vertex_intents.items() if "label" in intents or "useAsEndpoint" in intents} + if allow or not changed: + return + labels: dict[str, list[str]] = defaultdict(list) + for record in scope.vertices: + expected = work.expected_vertices.get(record.id, {}) + if expected.get("use_as_endpoint", record.use_as_endpoint): + label = expected.get("label", record.label) + if label: + labels[label].append(record.id) + for vertex_id, label in scope.external_endpoint_labels.items(): + labels[label].append(vertex_id) + collisions = {label: ids for label, ids in labels.items() if len(ids) > 1 and changed.intersection(ids)} + if collisions: + detail = "; ".join(f"'{label}' on {', '.join(sorted(ids))}" for label, ids in sorted(collisions.items())) + raise BlueprintValidationError( + ValidationIssue( + path="naming.endpoint_label", + message=f"Endpoint label collision in device '{scope.device_id}': {detail}. Adjust the naming rule " + "or allow duplicates with ApplyOptions(naming_collisions='allow').", + code="naming.collision", + ) + ) + + +def _select_override(override: Any, scope: ScopeData, module_position: str | None, index: int) -> str: + if override.vertex_id is not None: + if scope.vertex(override.vertex_id) is None: + raise BlueprintTargetError( + f"Vertex override #{index}: '{override.vertex_id}' is not in {_scope_label(scope)}." + ) + return override.vertex_id + label = _substitute_position(override.factory_label, module_position, f"vertices[{index}]") + matches = [ + vertex + for vertex in scope.vertices + if vertex.factory_label == label + and (override.kind is None or vertex.kind == override.kind) + and (override.direction is None or vertex.vertex_type == override.direction) + ] + if len(matches) != 1: + found = ", ".join(v.id for v in matches) or "none" + raise BlueprintTargetError( + f"Vertex override #{index}: factory label '{label}' must match exactly one vertex in " + f"{_scope_label(scope)}; matched {found}. Add 'kind'/'direction' or use 'vertex_id'." + ) + return matches[0].id + + +def _substitute_position(template: str, module_position: str | None, owner: str) -> str: + def replace(match: re.Match[str]) -> str: + if match.group(1) != "module.position": + raise BlueprintValidationError( + ValidationIssue( + path=owner, + message=f"Unsupported placeholder '{{{match.group(1)}}}' (only '{{module.position}}').", + code="mapping.placeholder", + ) + ) + if module_position is None: + raise BlueprintValidationError( + ValidationIssue( + path=owner, + message="'{module.position}' requires BlueprintDevice.module_position.", + code="mapping.missing", + ) + ) + return module_position + + escaped = template.replace("{{", "\x00").replace("}}", "\x01") + return _POSITION_TOKEN.sub(replace, escaped).replace("\x00", "{").replace("\x01", "}") + + +def _resolve_tags(spec: Any, current: Any, owner: str) -> list[str]: + current_list = list(current or []) + if isinstance(spec, TagDelta): + kept = [tag for tag in current_list if tag not in spec.remove] + return kept + [tag for tag in dict.fromkeys(spec.add) if tag not in kept] + if isinstance(spec, (list, tuple)): + return list(dict.fromkeys(spec)) + raise BlueprintValidationError(f"Invalid tag specification for {owner}.") + + +def _same(field: str, current: Any, desired: Any) -> bool: + if field == "tags": + return set(current or ()) == set(desired or ()) + return _plain(current) == _plain(desired) + + +def _device_value(device: DeviceRecord, field: str) -> Any: + return getattr(device, field) + + +def _wire(field: str, value: Any) -> Any: + if field == "coordinates": + return {"x": value[0], "y": value[1]} + if field == "tags": + return list(value) + return value + + +def _plain(value: Any) -> Any: + if isinstance(value, Coordinates): + return (float(value.x), float(value.y)) + if isinstance(value, tuple): + return tuple(_plain(item) for item in value) + return value + + +def _json(value: Any) -> Any: + value = _plain(value) + return list(value) if isinstance(value, tuple) else value + + +def _scope_label(scope: ScopeData) -> str: + return ( + f"module '{scope.module_id}' of device '{scope.device_id}'" + if scope.module_id + else f"device '{scope.device_id}'" + ) + + +def _items(collection: Any) -> list[tuple[str | None, Any]]: + if not collection: + return [] + if isinstance(collection, dict): + return list(collection.items()) + return [(None, item) for item in collection] + + +def _modules_of(node: InspectApiNodeStatusItem | None) -> dict[str, Any]: + if node is None: + return {} + modules: dict[str, Any] = {} + for key, module in _items(node.modules): + module_id = module.pid or module.id or key + if module_id: + modules[module_id] = module + return modules + + +def _vertex_sides(info: Any) -> list[tuple[str, InspectApiSingleVertexInfo]]: + if isinstance(info, InspectApiSingleVertexInfo): + return [(info.id, info)] if info.id else [] + if isinstance(info, InspectApiDoubleVertexInfo): + return [(side.id, side) for side in (info.out, info.in_) if side is not None and side.id] + return [] + + +def _local_tags(module: Any) -> tuple[str, ...] | None: + info = module.tagsInfo + assigned = info.get("assigned") if isinstance(info, dict) else None + local = assigned.get("local") if isinstance(assigned, dict) else None + return tuple(local.keys()) if isinstance(local, dict) else None + + +def _module_record(module_id: str, module: Any) -> ModuleRecord: + return ModuleRecord(id=module_id, label=module.effective_label, local_tags=_local_tags(module)) + + +def _device_record(device_id: str, node: InspectApiNodeStatusItem | None, form: Any) -> DeviceRecord: + factory = node.fDescriptor.label if node is not None and node.fDescriptor is not None else None + if form is None: + return DeviceRecord(id=device_id, factory_label=factory) + coordinates = form.coordinates + return DeviceRecord( + id=device_id, + label=form.descriptor.label, + description=form.descriptor.desc, + factory_label=factory, + icon_type=form.iconType, + icon_size=form.iconSize, + sdp_strategy=form.sdpStrategy, + site_id=form.siteId, + coordinates=(float(coordinates["x"]), float(coordinates["y"])) + if isinstance(coordinates, dict) and "x" in coordinates and "y" in coordinates + else None, + tags=tuple(form.localAssignedTags), + ) + + +def _vertex_record(vertex_id: str, side: InspectApiSingleVertexInfo, port: PortRecord, lookup: Any) -> VertexRecord: + base = { + "id": vertex_id, + "port_id": port.id, + "module_id": port.module_id, + "factory_label": port.factory_label, + "port_label": port.label, + "vertex_type": side.vertexType, + } + if lookup is None: + return VertexRecord(**base) + form = lookup.fields + type_fields = form.typeFields + generic = getattr(type_fields, "generic", None) if type_fields is not None else None + specific = getattr(type_fields, "specific", None) if type_fields is not None else None + return VertexRecord( + **base, + kind=type_fields.type if type_fields is not None else None, + label=form.label, + description=form.desc, + use_as_endpoint=form.useAsEndpoint, + active=form.active, + sips_mode=form.sipsMode, + tags=tuple(form.localAssignedTags), + codec_format=generic.get("codecFormat") if isinstance(generic, dict) else None, + media_type=specific.get("type") if isinstance(specific, dict) else None, + sdp_support=specific.get("sdpSupport") if isinstance(specific, dict) else None, + main_destination_port=_endpoint_port(generic, "mainDstInfo"), + spare_destination_port=_endpoint_port(generic, "spareDstInfo"), + supports_static_igmp=type_fields.supportsStaticIgmpCfg if type_fields is not None else None, + ) + + +def _endpoint_port(generic: Any, block: str) -> int | None: + info = generic.get(block) if isinstance(generic, dict) else None + port = info.get("port") if isinstance(info, dict) else None + return int(port) if isinstance(port, (int, float)) and not isinstance(port, bool) else None + + +def _check_action(response: Any, action: str, tag_id: str) -> None: + if response.header.ok and response.data.ok: + return + detail = "; ".join(m for m in list(response.data.msg) + list(response.header.msg) if m) or "rejected" + raise BlueprintError(f"{action} failed for tag '{tag_id}': {detail}") + + +__all__ = [ + "InspectGateway", + "ScopeData", + "TopologyWork", + "build_context", + "compute_topology_work", + "name_context", + "resolve_interfaces", + "verify_topology", +] diff --git a/src/videoipath_automation_tool/blueprints/inventory.py b/src/videoipath_automation_tool/blueprints/inventory.py new file mode 100644 index 0000000..3162f20 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/inventory.py @@ -0,0 +1,475 @@ +"""Inventory integration: plan and write exactly one Inventory record through the injected app. + +Only *managed* fields are compared and written: fields explicitly present in the resolved blueprint, +deliberately supplied instance facts, and labels from enabled naming rules. For an update the current +record is deep-copied and only managed fields are overlaid, so unrelated settings are preserved +(within what the Inventory models represent — the same round-trip ``InventoryApp.update_device`` +performs). The generic comparison of ``update_device`` is bypassed once a managed change is known, so +explicit credential changes cannot be swallowed by its authentication filtering. + +Conflict detection is client-side (re-read and compare before writing); a read-to-write race remains. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable +from typing import Any + +from pydantic import BaseModel, ConfigDict, SecretStr + +from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice +from videoipath_automation_tool.apps.inventory.model.inventory_device_configuration import Auth +from videoipath_automation_tool.blueprints.errors import ( + BlueprintCapabilityError, + BlueprintConflictError, + BlueprintTargetError, + BlueprintValidationError, + ValidationIssue, +) +from videoipath_automation_tool.blueprints.models import ( + AlternativeAddress, + BlueprintDevice, + CatalogId, + FieldChange, + PlannedOperation, + normalize_address, +) +from videoipath_automation_tool.blueprints.resolution import ResolvedInventory + +REDACTED = "********" +_SENSITIVE_TOKENS = frozenset({"password", "passwd", "secret", "token", "community", "key"}) + +# Fields whose change may alter what the driver discovers (topology work after them is deferred). +TOPOLOGY_AFFECTING_PREFIXES = ("address", "alt_addresses", "credentials.", "generic.", "custom_settings.", "active") + + +class DesiredValue(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + value: Any + source: str + + +class InventoryWork(BaseModel): + """Internal, unredacted Inventory work captured by a plan (never serialized for review).""" + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + action: str # "create" | "update" | "none" + inventory_id: str | None = None + driver_id: str + desired: dict[str, DesiredValue] + baseline: dict[str, str] + """Fingerprints of the current managed values (and driver) used for the stale-plan check.""" + changes: list[FieldChange] + + @property + def operation(self) -> PlannedOperation | None: + if self.action == "none": + return None + return PlannedOperation( + action="create" if self.action == "create" else "update", + entity_kind="inventory", + entity_id=self.inventory_id, + changes=self.changes, + ) + + @property + def affects_topology(self) -> bool: + return self.action == "create" or any( + change.field.startswith(TOPOLOGY_AFFECTING_PREFIXES) for change in self.changes + ) + + +class InventoryGateway: + """The only place that talks to ``app.inventory``.""" + + def __init__(self, inventory_app: Any) -> None: + self._app = inventory_app + self._snmp_cache: dict[str, str] = {} + + # --- Reads --- + + def read(self, inventory_id: str) -> InventoryDevice: + try: + return self._app.get_device(device_id=inventory_id, config_only=True) + except ValueError as exc: + raise BlueprintTargetError(f"Inventory record '{inventory_id}' was not found: {exc}") from None + + def ids_with_label(self, label: str) -> list[str]: + found = self._app.find_device_id_by_label(label, label_search_mode="user_defined_label_only") + return _as_list(found) + + def ids_with_address(self, address: str) -> list[str]: + candidates = {address.strip(), normalize_address(address)} + return self._app.find_device_ids_by_addresses(candidates) + + def resolve_snmp(self, reference: str | CatalogId) -> str: + """Exact SNMP configuration id for a label or ``{id: ...}``; missing/ambiguous fail.""" + key = reference.id if isinstance(reference, CatalogId) else f"label:{reference}" + if key in self._snmp_cache: + return self._snmp_cache[key] + if isinstance(reference, CatalogId): + if reference.id != "default" and self._app.get_global_snmp_config_label_by_id(reference.id) is None: + raise BlueprintTargetError(f"SNMP configuration id '{reference.id}' does not exist.") + resolved = reference.id + else: + found = _as_list(self._app.get_global_snmp_config_id_by_label(reference)) + if not found: + raise BlueprintTargetError(f"No SNMP configuration labelled '{reference}'.") + if len(found) > 1: + raise BlueprintTargetError( + f"SNMP configuration label '{reference}' is ambiguous ({', '.join(found)}); use {{id: ...}}." + ) + resolved = found[0] + self._snmp_cache[key] = resolved + return resolved + + # --- Writes --- + + def create(self, candidate: InventoryDevice) -> InventoryDevice: + return self._app.add_device(candidate, label_check=False, address_check=False, config_only=True) + + def update(self, candidate: InventoryDevice) -> InventoryDevice: + return self._app.update_device(candidate, compare_config=False, config_only=True) + + +def plan_inventory( + gateway: InventoryGateway, + resolved: ResolvedInventory, + device: BlueprintDevice, + *, + label: str | None, + description: str | None, +) -> tuple[InventoryWork, InventoryDevice | None]: + """Compute the Inventory work (read-only). Returns the work and the current record (if bound).""" + desired = _desired_values(gateway, resolved, device, label=label, description=description) + + current: InventoryDevice | None = None + if device.inventory_id is not None: + current = gateway.read(device.inventory_id) + if current.driver_id != resolved.driver_id: + raise BlueprintCapabilityError( + f"Inventory record '{device.inventory_id}' uses driver '{current.driver_id}', the blueprint selects " + f"'{resolved.driver_id}'. Driver migration is not automatic; migrate explicitly with the Inventory API." + ) + else: + if device.management_address is None: + raise BlueprintTargetError( + f"Creating an Inventory record for '{device.key}' requires 'management_address' " + "(or bind an existing record with 'inventory_id')." + ) + if "label" not in desired: + raise BlueprintValidationError( + ValidationIssue( + path="naming.inventory_label", + message="Creating an Inventory record requires an Inventory label; the naming entry is disabled.", + code="naming.missing", + ) + ) + + changes = [ + _field_change(key, get_field(current, key) if current is not None else None, item) + for key, item in desired.items() + if current is None or not values_equal(get_field(current, key), item.value) + ] + _check_conflicts(gateway, device, desired, changes, own_id=device.inventory_id) + + action = "create" if current is None else ("update" if changes else "none") + work = InventoryWork( + action=action, + inventory_id=device.inventory_id, + driver_id=resolved.driver_id, + desired=desired, + baseline=_baseline(current, desired) if current is not None else {}, + changes=changes, + ) + return work, current + + +def recheck_conflicts(gateway: InventoryGateway, device: BlueprintDevice, work: InventoryWork) -> None: + """Repeat the label/address conflict check immediately before a write.""" + _check_conflicts(gateway, device, work.desired, work.changes, own_id=work.inventory_id) + + +def build_candidate(base: InventoryDevice, desired: dict[str, DesiredValue]) -> InventoryDevice: + candidate = base.model_copy(deep=True) + for key, item in desired.items(): + set_field(candidate, key, item.value) + return candidate + + +def new_device(driver_id: str) -> InventoryDevice: + return InventoryDevice.create(driver_id=driver_id) + + +def check_baseline(work: InventoryWork, fresh: InventoryDevice) -> None: + """Stop when a managed value or the driver changed since planning.""" + current = _baseline(fresh, work.desired) + changed = sorted(key for key in work.baseline if current.get(key) != work.baseline[key]) + if changed: + raise BlueprintConflictError( + f"Inventory record '{work.inventory_id}' changed since planning ({', '.join(changed)}); replan." + ) + + +def verify(device: InventoryDevice, desired: dict[str, DesiredValue]) -> list[str]: + """Managed fields whose read-back differs (secrets are not compared: the server may mask them).""" + return sorted( + key + for key, item in desired.items() + if not is_sensitive(key) and not values_equal(get_field(device, key), item.value) + ) + + +def get_field(device: InventoryDevice, key: str) -> Any: + getter, _ = _accessor(key) + return getter(device) + + +def set_field(device: InventoryDevice, key: str, value: Any) -> None: + _, setter = _accessor(key) + setter(device, value) + + +def values_equal(current: Any, desired: Any) -> bool: + return _plain(current) == _plain(desired) + + +def is_sensitive(key: str) -> bool: + if key in ("credentials.password", "alt_addresses_with_auth"): + return True + tokens = set(re.split(r"[^a-z0-9]+", key.lower())) + return bool(tokens & _SENSITIVE_TOKENS) + + +# --- Internal --- + + +def _desired_values( + gateway: InventoryGateway, + resolved: ResolvedInventory, + device: BlueprintDevice, + *, + label: str | None, + description: str | None, +) -> dict[str, DesiredValue]: + config = resolved.config + provenance = resolved.provenance + desired: dict[str, DesiredValue] = {} + + def put(key: str, value: Any, source: str) -> None: + desired[key] = DesiredValue(value=value, source=source) + + if label is not None: + put("label", label, "naming") + if description is not None: + put("description", description, "naming") + if device.management_address is not None: + put("address", device.management_address, "instance") + if device.alternative_addresses is not None: + entries = device.alternative_addresses + put("alt_addresses", [e if isinstance(e, str) else e.address for e in entries], "instance") + put( + "alt_addresses_with_auth", + [_alt_with_auth(e) for e in entries if isinstance(e, AlternativeAddress) and e.credentials is not None], + "instance", + ) + if device.credentials is not None: + put("credentials.username", device.credentials.username, "instance") + put("credentials.password", device.credentials.password, "instance") + if config.active is not None: + put("active", config.active, provenance.get("active", "blueprint")) + if config.generic_settings is not None: + for name, value in config.generic_settings.managed().items(): + put(f"generic.{name}", value, provenance.get(f"generic_settings.{name}", "blueprint")) + if config.snmp is not None: + snmp = config.snmp.managed() + if "use_global_settings" in snmp: + put( + "snmp.use_global_settings", + snmp["use_global_settings"], + provenance.get("snmp.use_global_settings", "blueprint"), + ) + if "configuration" in snmp: + source = provenance.get("snmp.configuration", provenance.get("snmp.configuration.id", "blueprint")) + put("snmp.configuration", gateway.resolve_snmp(snmp["configuration"]), source) + for name, value in resolved.custom_settings.items(): + put(f"custom_settings.{name}", value, provenance.get(f"custom_settings.{name}", "blueprint")) + for name, value in (config.metadata or {}).items(): + put(f"metadata.{name}", value, provenance.get(f"metadata.{name}", "blueprint")) + return desired + + +def _alt_with_auth(entry: AlternativeAddress) -> dict[str, Any]: + assert entry.credentials is not None + return { + "address": entry.address, + "authentication": {"user": entry.credentials.username, "password": entry.credentials.password}, + } + + +def _check_conflicts( + gateway: InventoryGateway, + device: BlueprintDevice, + desired: dict[str, DesiredValue], + changes: list[FieldChange], + *, + own_id: str | None, +) -> None: + """Label/address collisions with *other* records for values that are being set or changed.""" + changed = {change.field for change in changes} + conflicts: list[str] = [] + if "label" in changed: + others = [i for i in gateway.ids_with_label(desired["label"].value) if i != own_id] + if others: + conflicts.append(f"label '{desired['label'].value}' is used by {', '.join(others)}") + addresses: list[str] = [] + if "address" in changed: + addresses.append(desired["address"].value) + if "alt_addresses" in changed: + addresses.extend(desired["alt_addresses"].value) + for address in addresses: + others = [i for i in gateway.ids_with_address(address) if i != own_id] + if others: + conflicts.append(f"address '{address}' is used by {', '.join(others)}") + if conflicts: + hint = "" if own_id else " Bind the existing record explicitly with 'inventory_id' if it is the same device." + raise BlueprintTargetError(f"Inventory conflict for '{device.key}': {'; '.join(conflicts)}.{hint}") + + +def _baseline(device: InventoryDevice, desired: dict[str, DesiredValue]) -> dict[str, str]: + baseline = {key: _digest(get_field(device, key)) for key in desired} + baseline["driver_id"] = _digest(device.driver_id) + return baseline + + +def _digest(value: Any) -> str: + return hashlib.sha256(json.dumps(_plain(value), sort_keys=True, default=str).encode()).hexdigest() + + +def _field_change(key: str, before: Any, item: DesiredValue) -> FieldChange: + sensitive = is_sensitive(key) + return FieldChange( + field=key, + before=_redact(key, before) if sensitive else _plain(before), + after=_redact(key, item.value) if sensitive else _plain(item.value), + source=item.source, + sensitive=sensitive, + ) + + +def _redact(key: str, value: Any) -> Any: + if value is None or value == "": + return value + if key == "alt_addresses_with_auth" and isinstance(value, list): + return [ + {"address": entry.get("address"), "authentication": REDACTED} for entry in value if isinstance(entry, dict) + ] + return REDACTED + + +def _plain(value: Any) -> Any: + if isinstance(value, SecretStr): + return value.get_secret_value() + if isinstance(value, dict): + return {key: _plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(item) for item in value] + if isinstance(value, BaseModel): + return _plain(value.model_dump(mode="json")) + return value + + +def _as_list(found: Any) -> list[str]: + if found is None: + return [] + return [found] if isinstance(found, str) else list(found) + + +def _auth(device: InventoryDevice) -> Auth: + cinfo = device.configuration.config.cinfo + if cinfo.auth is None: + cinfo.auth = Auth() + return cinfo.auth + + +def _set_password(device: InventoryDevice, value: Any) -> None: + _auth(device).password = value.get_secret_value() if isinstance(value, SecretStr) else value + + +def _set_alt_with_auth(device: InventoryDevice, value: list[dict[str, Any]]) -> None: + device.configuration.config.cinfo.altAddressesWithAuth = _plain(value) + + +def _accessor(key: str) -> tuple[Callable[[InventoryDevice], Any], Callable[[InventoryDevice, Any], None]]: + fixed = _ACCESSORS.get(key) + if fixed is not None: + return fixed + prefix, _, name = key.partition(".") + if prefix == "custom_settings": + return ( + lambda d: getattr(d.configuration.config.customSettings, name, None), + lambda d, v: setattr(d.configuration.config.customSettings, name, v), + ) + if prefix == "metadata": + return (lambda d: d.configuration.meta.get(name), lambda d, v: d.configuration.meta.__setitem__(name, v)) + raise KeyError(f"Unsupported managed Inventory field '{key}'.") + + +def _cinfo(device: InventoryDevice) -> Any: + return device.configuration.config.cinfo + + +_ACCESSORS: dict[str, tuple[Callable[[InventoryDevice], Any], Callable[[InventoryDevice, Any], None]]] = { + "label": ( + lambda d: d.configuration.config.desc.label, + lambda d, v: setattr(d.configuration.config.desc, "label", v), + ), + "description": ( + lambda d: d.configuration.config.desc.desc, + lambda d, v: setattr(d.configuration.config.desc, "desc", v), + ), + "active": (lambda d: d.configuration.active, lambda d, v: setattr(d.configuration, "active", v)), + "address": (lambda d: _cinfo(d).address, lambda d, v: setattr(_cinfo(d), "address", v)), + "alt_addresses": (lambda d: list(_cinfo(d).altAddresses), lambda d, v: setattr(_cinfo(d), "altAddresses", list(v))), + "alt_addresses_with_auth": (lambda d: list(_cinfo(d).altAddressesWithAuth), _set_alt_with_auth), + "credentials.username": ( + lambda d: _cinfo(d).auth.user if _cinfo(d).auth is not None else None, + lambda d, v: setattr(_auth(d), "user", v), + ), + "credentials.password": (lambda d: _cinfo(d).auth.password if _cinfo(d).auth is not None else None, _set_password), + "generic.enable_https": (lambda d: _cinfo(d).http.https, lambda d, v: setattr(_cinfo(d).http, "https", v)), + "generic.trust_all_certificates": ( + lambda d: _cinfo(d).http.trustAllCertificates, + lambda d, v: setattr(_cinfo(d).http, "trustAllCertificates", v), + ), + "generic.http_auth": (lambda d: _cinfo(d).http.httpAuth, lambda d, v: setattr(_cinfo(d).http, "httpAuth", v)), + "snmp.use_global_settings": ( + lambda d: d.configuration.cinfoOverrides.snmp.useDefault, + lambda d, v: setattr(d.configuration.cinfoOverrides.snmp, "useDefault", v), + ), + "snmp.configuration": ( + lambda d: d.configuration.cinfoOverrides.snmp.id, + lambda d, v: setattr(d.configuration.cinfoOverrides.snmp, "id", v), + ), +} + + +__all__ = [ + "InventoryGateway", + "InventoryWork", + "build_candidate", + "check_baseline", + "get_field", + "is_sensitive", + "new_device", + "plan_inventory", + "recheck_conflicts", + "set_field", + "values_equal", + "verify", +] diff --git a/src/videoipath_automation_tool/blueprints/loader.py b/src/videoipath_automation_tool/blueprints/loader.py new file mode 100644 index 0000000..b19e2fb --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/loader.py @@ -0,0 +1,186 @@ +"""Strict YAML loading for blueprints. + +Starts from PyYAML's safe loading (a dedicated ``SafeLoader`` subclass; PyYAML is never modified +globally) and adds the project's requirements: + +- exactly one UTF-8 document with a mapping root, within size and depth limits; +- duplicate keys rejected at any depth; mapping keys must be strings; +- no aliases, anchors, merge keys, or custom tags; +- implicit scalars restricted to ``true``/``false``, ``null``/``~``, decimal integers, and plain + decimal floats — ``off``, ``yes``, timestamps, octal/sexagesimal numbers stay strings; +- every node's line/column is recorded so validation errors point at the source. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, ClassVar + +import yaml + +from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue + +MAX_DOCUMENT_BYTES = 1024 * 1024 +MAX_DEPTH = 32 +MAX_NODES = 100_000 + + +def read_blueprint_file(path: Path) -> bytes: + """Read a blueprint file, enforcing the size limit before parsing.""" + with path.open("rb") as handle: + data = handle.read(MAX_DOCUMENT_BYTES + 1) + if len(data) > MAX_DOCUMENT_BYTES: + raise _error(f"document exceeds {MAX_DOCUMENT_BYTES} bytes", str(path)) + return data + + +def parse_yaml( + text: str | bytes, *, source: str | None = None +) -> tuple[dict[str, Any], dict[tuple[Any, ...], tuple[int, int]]]: + """Parse one strict YAML document; returns ``(data, locations)`` where ``locations`` maps each + document path to its 1-based ``(line, column)``.""" + if isinstance(text, bytes): + if len(text) > MAX_DOCUMENT_BYTES: + raise _error(f"document exceeds {MAX_DOCUMENT_BYTES} bytes", source) + try: + text = text.decode("utf-8") + except UnicodeDecodeError as exc: + raise _error(f"document is not valid UTF-8 ({exc.reason} at byte {exc.start})", source) from None + elif len(text.encode("utf-8")) > MAX_DOCUMENT_BYTES: + raise _error(f"document exceeds {MAX_DOCUMENT_BYTES} bytes", source) + text = text.removeprefix("") + + loader = _StrictLoader(text) + try: + node = loader.get_single_node() + except _LoaderError as exc: + raise _error(exc.message, source, exc.mark) from None + except RecursionError: + raise _error(f"document nesting exceeds depth {MAX_DEPTH}", source) from None + except yaml.YAMLError as exc: + mark = getattr(exc, "problem_mark", None) + problem = getattr(exc, "problem", None) or str(exc) + raise _error(f"invalid YAML: {problem}", source, mark) from None + finally: + loader.dispose() + + if node is None: + raise _error("document is empty", source) + builder = _Builder(source) + data = builder.build(node, (), depth=0) + if not isinstance(data, dict): + raise _error("the document root must be a mapping", source, node.start_mark) + return data, builder.locations + + +# --- Internal --- + +_BOOL_RE = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +_NULL_RE = re.compile(r"^(?:~|null|Null|NULL|)$") +_INT_RE = re.compile(r"^[-+]?(?:0|[1-9][0-9]*)$") +_FLOAT_RE = re.compile(r"^[-+]?(?:[0-9]+\.[0-9]*|\.[0-9]+)(?:[eE][-+]?[0-9]+)?$|^[-+]?[0-9]+[eE][-+]?[0-9]+$") + +_TAG_STR = "tag:yaml.org,2002:str" +_TAG_BOOL = "tag:yaml.org,2002:bool" +_TAG_NULL = "tag:yaml.org,2002:null" +_TAG_INT = "tag:yaml.org,2002:int" +_TAG_FLOAT = "tag:yaml.org,2002:float" +_TAG_MAP = "tag:yaml.org,2002:map" +_TAG_SEQ = "tag:yaml.org,2002:seq" + + +class _LoaderError(Exception): + def __init__(self, message: str, mark: Any) -> None: + self.message = message + self.mark = mark + super().__init__(message) + + +class _StrictLoader(yaml.SafeLoader): + """Safe loader with restricted implicit scalars that rejects aliases and anchors.""" + + yaml_implicit_resolvers: ClassVar[dict[str, list[Any]]] = {} + + def compose_node(self, parent: Any, index: Any) -> Any: + if self.check_event(yaml.AliasEvent): + raise _LoaderError("aliases are not supported", self.peek_event().start_mark) + event = self.peek_event() + if getattr(event, "anchor", None) is not None: + raise _LoaderError("anchors are not supported", event.start_mark) + return super().compose_node(parent, index) + + +for _tag, _regexp, _first in ( + (_TAG_BOOL, _BOOL_RE, list("tTfF")), + (_TAG_NULL, _NULL_RE, ["~", "n", "N", ""]), + (_TAG_INT, _INT_RE, list("-+0123456789")), + (_TAG_FLOAT, _FLOAT_RE, list("-+.0123456789")), +): + _StrictLoader.add_implicit_resolver(_tag, _regexp, _first) + + +class _Builder: + """Constructs plain Python data from the composed node tree, enforcing the strict rules.""" + + def __init__(self, source: str | None) -> None: + self._source = source + self._nodes = 0 + self.locations: dict[tuple[Any, ...], tuple[int, int]] = {} + + def build(self, node: Any, path: tuple[Any, ...], depth: int) -> Any: + self._nodes += 1 + if self._nodes > MAX_NODES: + raise _error(f"document exceeds {MAX_NODES} nodes", self._source, node.start_mark) + if depth > MAX_DEPTH: + raise _error(f"document nesting exceeds depth {MAX_DEPTH}", self._source, node.start_mark) + self.locations[path] = (node.start_mark.line + 1, node.start_mark.column + 1) + + if isinstance(node, yaml.MappingNode): + if node.tag != _TAG_MAP: + raise _error(f"unsupported tag {node.tag!r}", self._source, node.start_mark) + result: dict[str, Any] = {} + for key_node, value_node in node.value: + if key_node.tag == "tag:yaml.org,2002:merge" or (key_node.value == "<<" and key_node.style is None): + raise _error("merge keys ('<<') are not supported", self._source, key_node.start_mark) + if not isinstance(key_node, yaml.ScalarNode) or key_node.tag != _TAG_STR: + raise _error("mapping keys must be strings", self._source, key_node.start_mark) + key = key_node.value + if key in result: + raise _error(f"duplicate key {key!r}", self._source, key_node.start_mark) + result[key] = self.build(value_node, path + (key,), depth + 1) + return result + if isinstance(node, yaml.SequenceNode): + if node.tag != _TAG_SEQ: + raise _error(f"unsupported tag {node.tag!r}", self._source, node.start_mark) + return [self.build(item, path + (index,), depth + 1) for index, item in enumerate(node.value)] + return self._scalar(node) + + def _scalar(self, node: Any) -> Any: + tag, value = node.tag, node.value + if tag == _TAG_STR: + return value + if tag == _TAG_NULL and _NULL_RE.match(value): + return None + if tag == _TAG_BOOL and _BOOL_RE.match(value): + return value.lower() == "true" + if tag == _TAG_INT and _INT_RE.match(value): + return int(value) + if tag == _TAG_FLOAT and _FLOAT_RE.match(value): + return float(value) + raise _error(f"unsupported scalar tag or value for tag {tag!r}", self._source, node.start_mark) + + +def _error(message: str, source: str | None, mark: Any = None) -> BlueprintValidationError: + return BlueprintValidationError( + ValidationIssue( + message=message, + code="yaml.invalid", + source=source, + line=mark.line + 1 if mark is not None else None, + column=mark.column + 1 if mark is not None else None, + ) + ) + + +__all__ = ["MAX_DEPTH", "MAX_DOCUMENT_BYTES", "MAX_NODES", "parse_yaml", "read_blueprint_file"] diff --git a/src/videoipath_automation_tool/blueprints/models.py b/src/videoipath_automation_tool/blueprints/models.py new file mode 100644 index 0000000..95cc04e --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/models.py @@ -0,0 +1,688 @@ +"""Public blueprint models: the YAML document, source-independent device facts, typed patches and +processor output, and the plan/result records. + +Document models are strict (``extra="forbid"``, no implicit coercion) and immutable. Open mappings +exist only where intentional: driver ``custom_settings`` (validated against the selected driver at +resolution), processor ``params`` (validated by the registered processor), ``metadata``, and caller +``attributes``. Patch fields use *absence* to mean "unmanaged"; ``null`` is rejected there because it +would be ambiguous with absence. +""" + +from __future__ import annotations + +import ipaddress +from os import PathLike +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Any, Literal + +from pydantic import ( + AfterValidator, + BaseModel, + ConfigDict, + Field, + JsonValue, + PrivateAttr, + SecretStr, + StringConstraints, + ValidationError, + model_validator, +) + +from videoipath_automation_tool.apps.inspect.model.common import ( + InspectIconSize, + InspectIconType, + InspectSdpStrategy, + InspectSipsMode, +) +from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue +from videoipath_automation_tool.blueprints.loader import parse_yaml, read_blueprint_file +from videoipath_automation_tool.blueprints.naming import ( + INVENTORY_NAMING_ENTRIES, + TOPOLOGY_NAMING_ENTRIES, + BlueprintNaming, +) +from videoipath_automation_tool.validators.device_id import validate_device_id + +if TYPE_CHECKING: + from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice + from videoipath_automation_tool.blueprints.processors import ProcessorRegistry + +NonEmptyStr = Annotated[str, StringConstraints(min_length=1)] +VariantName = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")] +MetadataValue = str | int | float | bool +Scope = Literal["all", "inventory", "topology"] +SyncPolicy = Literal["none", "add_only", "reconcile"] + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + + +class _PatchModel(_StrictModel): + """Absent fields are unmanaged; explicit ``null`` is rejected (ambiguous with absence).""" + + @model_validator(mode="before") + @classmethod + def _reject_nulls(cls, data: Any) -> Any: + if isinstance(data, dict): + nulls = sorted(str(key) for key, value in data.items() if value is None) + if nulls: + raise ValueError(f"null is not supported for {', '.join(nulls)}; omit the field to leave it unmanaged") + return data + + def managed(self) -> dict[str, Any]: + """The explicitly supplied fields and their values.""" + return {name: getattr(self, name) for name in type(self).model_fields if name in self.model_fields_set} + + +# --- Tags --- + + +class TagDelta(_StrictModel): + """Additive tag change that preserves unrelated local tags. Tag references are exact catalog + tag ids (for example ``Format~~V_1080p50``); the engine never creates catalog entries.""" + + add: list[NonEmptyStr] = Field(default_factory=list) + remove: list[NonEmptyStr] = Field(default_factory=list) + + @model_validator(mode="after") + def _no_overlap(self) -> TagDelta: + overlap = sorted(set(self.add) & set(self.remove)) + if overlap: + raise ValueError(f"tags listed in both add and remove: {', '.join(overlap)}") + if not self.add and not self.remove: + raise ValueError("a tag delta needs at least one 'add' or 'remove' entry") + return self + + +TagSpec = list[NonEmptyStr] | TagDelta +"""A list replaces the local tags exactly (``[]`` clears them); ``{add, remove}`` changes them additively.""" + + +# --- Topology patches (YAML and processor output share these contracts) --- + + +class Coordinates(_StrictModel): + x: float + y: float + + +class DevicePatch(_PatchModel): + """Inspect device fields (device targets only). Device label/description come from naming.""" + + icon_type: InspectIconType | None = None + icon_size: InspectIconSize | None = None + sdp_strategy: InspectSdpStrategy | None = None + site_id: NonEmptyStr | None = None + coordinates: Coordinates | None = None + tags: TagSpec | None = None + + +class ModulePatch(_PatchModel): + """Module-local tag settings (module targets only).""" + + tags: TagSpec | None = None + + +class VertexPatch(_PatchModel): + """Supported vertex edits. ``""`` is an explicit clear for label/description. + + Kind-specific fields: ``sdp_support`` and the destination ports apply to codec vertices, + ``supports_static_igmp`` to IP vertices. + """ + + use_as_endpoint: bool | None = None + label: str | None = None + description: str | None = None + active: bool | None = None + sips_mode: InspectSipsMode | None = None + tags: TagSpec | None = None + sdp_support: bool | None = None + main_destination_port: Annotated[int, Field(ge=0, le=65535)] | None = None + spare_destination_port: Annotated[int, Field(ge=0, le=65535)] | None = None + supports_static_igmp: bool | None = None + + +class VertexOverride(_StrictModel): + """Explicit vertex edit. Select by exact ``vertex_id``, or by exact ``factory_label`` (supports + ``{module.position}``) optionally restricted by ``kind`` / ``direction``; must match exactly one vertex.""" + + vertex_id: NonEmptyStr | None = None + factory_label: NonEmptyStr | None = None + kind: NonEmptyStr | None = None + direction: Literal["In", "Out", "Internal", "Undecided"] | None = None + fields: VertexPatch + + @model_validator(mode="after") + def _one_selector(self) -> VertexOverride: + if (self.vertex_id is None) == (self.factory_label is None): + raise ValueError("specify exactly one of 'vertex_id' or 'factory_label'") + if self.vertex_id is not None and (self.kind is not None or self.direction is not None): + raise ValueError("'kind' / 'direction' only refine a 'factory_label' selector") + return self + + +class VertexProcessorSpec(_StrictModel): + """Processor declaration. ``processor_type`` may be inherited from ``default`` by a variant; + it is required on the resolved configuration.""" + + processor_type: NonEmptyStr | None = None + params: dict[str, Any] = Field(default_factory=dict) + + +# --- Inventory configuration --- + + +class CatalogId(_StrictModel): + """Exact catalog id reference, e.g. ``{id: default}`` for the system SNMP configuration.""" + + id: NonEmptyStr + + +class SnmpSelection(_PatchModel): + """Global SNMP selection. ``configuration`` is an exact label (string) or ``{id: ...}``.""" + + use_global_settings: bool | None = None + configuration: NonEmptyStr | CatalogId | None = None + + +class GenericSettings(_PatchModel): + """Generic connection settings (``cinfo.http``). ``http_auth`` is the server's raw + ``httpAuth`` mode code (``0`` disables HTTP authentication).""" + + enable_https: bool | None = None + trust_all_certificates: bool | None = None + http_auth: Annotated[int, Field(ge=0)] | None = None + + +class InventorySettings(_StrictModel): + """Inventory settings that may also be supplied per instance (``BlueprintDevice.inventory_overrides``).""" + + active: bool | None = None + generic_settings: GenericSettings | None = None + snmp: SnmpSelection | None = None + custom_settings: dict[str, Any] | None = None + metadata: dict[NonEmptyStr, MetadataValue] | None = None + + +class InventoryConfig(InventorySettings): + """One Inventory entry (``default`` or a named variant patch).""" + + driver_id: NonEmptyStr | None = None + naming: BlueprintNaming | None = None + + @model_validator(mode="after") + def _inventory_naming_only(self) -> InventoryConfig: + _check_naming_entries(self.naming, INVENTORY_NAMING_ENTRIES, "inventory") + if self.custom_settings is not None and "driver_id" in self.custom_settings: + raise ValueError("'driver_id' belongs at the inventory entry level, not inside custom_settings") + return self + + +class TopologyConfig(_StrictModel): + """One topology entry (``default`` or a named variant patch).""" + + device: DevicePatch | None = None + module: ModulePatch | None = None + ip_vertex_mapping: dict[NonEmptyStr, Annotated[list[NonEmptyStr], Field(min_length=1)]] | None = None + vertex_processor: VertexProcessorSpec | None = None + naming: BlueprintNaming | None = None + vertices: list[VertexOverride] | None = None + + @model_validator(mode="after") + def _topology_naming_only(self) -> TopologyConfig: + _check_naming_entries(self.naming, TOPOLOGY_NAMING_ENTRIES, "topology") + return self + + +class Blueprint(_StrictModel): + """A reusable, type-level configuration with ``default`` entries and optional named variants. + + Load with :meth:`load` (file), :meth:`from_yaml` (text/bytes), or :meth:`from_dict` (mapping). + Loading never connects to a server. A named variant is a patch over its section's ``default``. + """ + + schema_version: Literal[1] + inventory: dict[VariantName, InventoryConfig] | None = None + topology: dict[VariantName, TopologyConfig] | None = None + + _source: str | None = PrivateAttr(default=None) + _locations: dict[tuple[Any, ...], tuple[int, int]] = PrivateAttr(default_factory=dict) + + @model_validator(mode="after") + def _sections(self) -> Blueprint: + if self.inventory is None and self.topology is None: + raise ValueError("a blueprint needs an 'inventory' or a 'topology' section") + for name, section in (("inventory", self.inventory), ("topology", self.topology)): + if section is not None and "default" not in section: + raise ValueError(f"the '{name}' section must contain a 'default' entry") + return self + + # --- Construction --- + + @classmethod + def load(cls, path: str | PathLike[str]) -> Blueprint: + """Load one UTF-8 YAML blueprint file.""" + path = Path(path) + return cls.from_yaml(read_blueprint_file(path), source=str(path)) + + @classmethod + def from_yaml(cls, text: str | bytes, *, source: str | None = None) -> Blueprint: + """Parse YAML text with strict, safe loading.""" + data, locations = parse_yaml(text, source=source) + return cls._from_data(data, source=source, locations=locations) + + @classmethod + def from_dict(cls, data: dict[str, Any], *, source: str | None = None) -> Blueprint: + """Build from a mapping (e.g. produced by caller preprocessing). The mapping is not retained.""" + if not isinstance(data, dict): + raise BlueprintValidationError(ValidationIssue(message="a blueprint must be a mapping", source=source)) + return cls._from_data(data, source=source, locations={}) + + # --- Validation and schema --- + + def validate_full(self, registry: ProcessorRegistry | None = None) -> None: + """Resolve and validate *every* variant (including unused ones) against the selected driver + models, naming rules, and — when ``registry`` is given — processor parameter schemas. + Processors absent from ``registry`` are reported as ``processor.unavailable`` issues. + + Raises: + BlueprintValidationError: listing all issues found. + """ + from videoipath_automation_tool.blueprints.resolution import validate_all_variants + + validate_all_variants(self, registry) + + @classmethod + def json_schema( + cls, registry: ProcessorRegistry | None = None, *, restrict_processor_types: bool = False + ) -> dict[str, Any]: + """JSON Schema of the document; with ``registry``, adds parameter schemas per processor id. + ``restrict_processor_types`` additionally limits ``processor_type`` to the registered ids.""" + from videoipath_automation_tool.blueprints.resolution import blueprint_json_schema + + return blueprint_json_schema(registry, restrict_processor_types=restrict_processor_types) + + def variant_names(self, section: Literal["inventory", "topology"]) -> list[str]: + entries = getattr(self, section) + return list(entries) if entries else [] + + def location_of(self, path: tuple[Any, ...]) -> tuple[str | None, int | None, int | None]: + """``(source, line, column)`` of a document path when loaded from YAML.""" + line_col = self._locations.get(tuple(path)) + return self._source, (line_col[0] if line_col else None), (line_col[1] if line_col else None) + + # --- Internal --- + + @classmethod + def _from_data( + cls, + data: dict[str, Any], + *, + source: str | None, + locations: dict[tuple[Any, ...], tuple[int, int]], + ) -> Blueprint: + try: + blueprint = cls.model_validate(data) + except ValidationError as exc: + raise BlueprintValidationError( + issues_from_pydantic(exc, data=data, source=source, locations=locations) + ) from None + blueprint._source = source + blueprint._locations = dict(locations) + return blueprint + + +# --- Source-independent device facts --- + + +class Credentials(_StrictModel): + """Runtime device credentials (never part of YAML; redacted from reprs and plans).""" + + username: str + password: SecretStr + + +class AlternativeAddress(_StrictModel): + address: NonEmptyStr + credentials: Credentials | None = None + + +class DeviceTarget(_StrictModel): + """Configure the topology of an exact Inspect device.""" + + device_id: NonEmptyStr + + +class ModuleTarget(_StrictModel): + """Configure only one exact module of an Inspect device (ids come from Inspect, never labels).""" + + device_id: NonEmptyStr + module_id: NonEmptyStr + + +TopologyTarget = DeviceTarget | ModuleTarget + + +class BlueprintDevice(_StrictModel): + """Instance facts and explicit VideoIPath bindings for one source entity. + + ``key`` is an opaque caller correlation key (never written to VideoIPath). ``inventory_id`` binds + this entity's *own* Inventory record; without it, a selected inventory section creates one. + ``topology`` binds an exact Inspect device or module; without it the Inventory id is used. + ``alternative_addresses``: omitted = unmanaged, ``[]`` = clear. A supplied list is the full desired + set and also replaces per-address credentials: a plain string carries none, so an existing + credential for that address is cleared unless it is repeated as an ``AlternativeAddress``. + ``attributes`` are caller-owned facts available to naming (scalar leaves) and processors. + """ + + key: NonEmptyStr + label: NonEmptyStr + description: str | None = None + inventory_id: Annotated[str, AfterValidator(validate_device_id)] | None = None + management_address: NonEmptyStr | None = None + alternative_addresses: list[NonEmptyStr | AlternativeAddress] | None = None + credentials: Credentials | None = None + topology: TopologyTarget | None = None + module_position: NonEmptyStr | None = None + inventory_overrides: InventorySettings | None = None + attributes: dict[str, JsonValue] = Field(default_factory=dict) + + @model_validator(mode="after") + def _unique_addresses(self) -> BlueprintDevice: + seen: dict[str, Any] = {} + if self.management_address is not None: + seen[normalize_address(self.management_address)] = None + for entry in self.alternative_addresses or []: + address = entry if isinstance(entry, str) else entry.address + normalized = normalize_address(address) + if normalized in seen: + raise ValueError(f"address {address!r} is listed more than once") + seen[normalized] = entry + return self + + @classmethod + def from_inventory(cls, device: InventoryDevice, *, key: str | None = None) -> BlueprintDevice: + """Convenience binding for an existing Inventory device (id, label, description only).""" + configuration = device.configuration + return cls( + key=key or configuration.id, + label=configuration.config.desc.label or configuration.id, + description=configuration.config.desc.desc or None, + inventory_id=configuration.id, + ) + + +# --- Processor output contract --- + + +class EndpointIdentity(_StrictModel): + """Semantic endpoint facts for naming (never a company-specific label).""" + + direction: NonEmptyStr + media: NonEmptyStr + index: Annotated[int, Field(ge=0)] + engine: Annotated[int, Field(ge=0)] | None = None + program: Annotated[int, Field(ge=0)] | None = None + leg: NonEmptyStr | None = None + + +class VertexEdit(_StrictModel): + """Proposed edits for one exact vertex id within the processing scope.""" + + vertex_id: NonEmptyStr + fields: VertexPatch = Field(default_factory=VertexPatch) + endpoint: EndpointIdentity | None = None + + +class Diagnostic(_StrictModel): + level: Literal["info", "warning"] = "info" + code: NonEmptyStr + message: str + entity_id: str | None = None + + +class ProcessorResult(_StrictModel): + vertices: list[VertexEdit] = Field(default_factory=list) + device: DevicePatch | None = None + module: ModulePatch | None = None + diagnostics: list[Diagnostic] = Field(default_factory=list) + + +# --- Options --- + + +class ApplyOptions(_StrictModel): + """Execution options. + + ``sync``: ``"none"`` requires existing topology; ``"add_only"`` (default) adds/syncs only new + elements; ``"reconcile"`` permits a full sync. Service conflicts are always handled strictly. + ``discovery_timeout`` / ``poll_interval`` bound the wait for driver discovery (seconds). + ``naming_collisions="allow"`` permits duplicate endpoint labels within a device. + """ + + sync: SyncPolicy = "add_only" + discovery_timeout: Annotated[float, Field(gt=0)] = 30.0 + poll_interval: Annotated[float, Field(gt=0)] = 1.0 + naming_collisions: Literal["reject", "allow"] = "reject" + + +# --- Plan and result records --- + +PhaseName = Literal["inventory", "discovery", "topology_sync", "topology", "module_tags", "verification"] + + +class FieldChange(BaseModel): + """One managed field: ``before`` / ``after`` are redacted when ``sensitive``.""" + + model_config = ConfigDict(frozen=True) + + field: str + before: Any = None + after: Any = None + source: str = "" + sensitive: bool = False + + +class PlannedOperation(BaseModel): + model_config = ConfigDict(frozen=True) + + action: Literal["create", "update", "assign_tag", "unassign_tag", "add_to_topology", "sync"] + entity_kind: Literal["inventory", "device", "vertex", "module"] + entity_id: str | None = None + changes: list[FieldChange] = Field(default_factory=list) + endpoint: EndpointIdentity | None = None + + +class PlannedPhase(BaseModel): + model_config = ConfigDict(frozen=True) + + name: PhaseName + status: Literal["planned", "no_change", "deferred", "skipped"] + reason: str | None = None + operations: list[PlannedOperation] = Field(default_factory=list) + + +class InterfaceBinding(BaseModel): + """A caller interface key resolved to an exact port and its directional vertices.""" + + model_config = ConfigDict(frozen=True) + + key: str + candidate: str + port_id: str + in_vertex_id: str | None = None + out_vertex_id: str | None = None + + +class PhaseResult(BaseModel): + name: PhaseName + status: Literal["completed", "no_change", "skipped", "failed", "unknown", "not_run", "planned", "deferred"] + """``planned`` / ``deferred`` occur only in dry runs (writes that would run / work that needs earlier writes).""" + message: str | None = None + operations: list[PlannedOperation] = Field(default_factory=list) + + +class ApplyResult(BaseModel): + """What actually happened. Known ids are always reported, including after a failure. + + In a dry run (``dry_run=True``) nothing is written: ``status`` is ``planned`` when writes would + run and ``no_change`` otherwise, and write phases report ``planned`` / ``deferred``. + """ + + status: Literal["succeeded", "no_change", "failed", "partial", "unknown", "planned"] = "no_change" + dry_run: bool = False + source_key: str + inventory_id: str | None = None + topology_device_id: str | None = None + module_id: str | None = None + phases: list[PhaseResult] = Field(default_factory=list) + interface_bindings: list[InterfaceBinding] = Field(default_factory=list) + materialized: bool = False + diagnostics: list[Diagnostic] = Field(default_factory=list) + verification: Literal["confirmed", "unconfirmed", "not_applicable"] = "not_applicable" + verification_detail: str | None = None + + @property + def ok(self) -> bool: + return self.status in ("succeeded", "no_change", "planned") + + def phase(self, name: PhaseName) -> PhaseResult | None: + return next((phase for phase in self.phases if phase.name == name), None) + + +# --- Helpers --- + + +def normalize_address(address: str) -> str: + """Normalize IP literals (no DNS); other identifiers compare case-insensitively.""" + candidate = address.strip() + try: + return ipaddress.ip_address(candidate.strip("[]")).compressed + except ValueError: + return candidate.casefold() + + +def issues_from_pydantic( + exc: ValidationError, + *, + data: Any = None, + prefix: tuple[Any, ...] = (), + source: str | None = None, + locations: dict[tuple[Any, ...], tuple[int, int]] | None = None, +) -> list[ValidationIssue]: + """Convert a pydantic error to issues with document paths and YAML locations (no input values). + + ``data`` is the validated input; it is used to drop union-member tags pydantic inserts into + error locations.""" + issues: list[ValidationIssue] = [] + for error in exc.errors(include_input=False, include_url=False): + loc = prefix + _document_loc(tuple(error["loc"]), data) + line_col = _nearest_location(loc, locations or {}) + issues.append( + ValidationIssue( + path=".".join(str(part) for part in loc), + message=_friendly_message(error), + code=f"schema.{error['type']}", + source=source, + line=line_col[0] if line_col else None, + column=line_col[1] if line_col else None, + ) + ) + return issues + + +def _check_naming_entries(naming: BlueprintNaming | None, allowed: frozenset[str], section: str) -> None: + if naming is None: + return + invalid = sorted(name for name in naming.model_fields_set if name not in allowed) + if invalid: + raise ValueError( + f"naming entries {', '.join(invalid)} are not valid in the '{section}' section " + f"(allowed: {', '.join(sorted(allowed))})" + ) + + +def _document_loc(loc: tuple[Any, ...], data: Any) -> tuple[Any, ...]: + """Walk ``data`` along ``loc``; parts that are not keys/indices (union-member tags) are dropped, + except a trailing one (the name of a missing field).""" + if data is None: + return loc + node = data + kept: list[Any] = [] + for index, part in enumerate(loc): + if ( + isinstance(node, dict) + and part in node + or isinstance(node, list) + and isinstance(part, int) + and 0 <= part < len(node) + ): + node = node[part] + elif index < len(loc) - 1: + continue + kept.append(part) + return tuple(kept) + + +def _nearest_location( + loc: tuple[Any, ...], locations: dict[tuple[Any, ...], tuple[int, int]] +) -> tuple[int, int] | None: + for end in range(len(loc), -1, -1): + found = locations.get(loc[:end]) + if found is not None: + return found + return None + + +def _friendly_message(error: Any) -> str: + kind = error["type"] + if kind == "bool_type": + return "Expected a boolean; received a different type." + if kind == "int_type": + return "Expected an integer; received a different type." + if kind == "string_type": + return "Expected a string; received a different type." + if kind == "extra_forbidden": + return "Unknown field." + if kind == "missing": + return "Field required." + message = str(error["msg"]) + return message.removeprefix("Value error, ") + + +__all__ = [ + "AlternativeAddress", + "ApplyOptions", + "ApplyResult", + "Blueprint", + "BlueprintDevice", + "CatalogId", + "Coordinates", + "Credentials", + "DevicePatch", + "DeviceTarget", + "Diagnostic", + "EndpointIdentity", + "FieldChange", + "GenericSettings", + "InterfaceBinding", + "InventoryConfig", + "InventorySettings", + "ModulePatch", + "ModuleTarget", + "PhaseResult", + "PlannedOperation", + "PlannedPhase", + "ProcessorResult", + "SnmpSelection", + "TagDelta", + "TagSpec", + "TopologyConfig", + "TopologyTarget", + "VertexEdit", + "VertexOverride", + "VertexPatch", + "VertexProcessorSpec", + "normalize_address", +] diff --git a/src/videoipath_automation_tool/blueprints/naming.py b/src/videoipath_automation_tool/blueprints/naming.py new file mode 100644 index 0000000..6478e34 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/naming.py @@ -0,0 +1,426 @@ +"""Composable naming for Inventory, topology device, and endpoint labels. + +A naming *entry* (``inventory_label``, ``endpoint_label``, …) holds one expression built from three +blocks — :class:`Text`, :class:`Field`, and :class:`Join` — or a template string such as +``"{device.label}-{endpoint.index:02d}"``, which compiles to the same blocks. Names are derived only +from explicit facts (device facts, module position, semantic endpoint identity, caller attributes), +never from a current label, so applying twice cannot append a second suffix. + +Formatting supports a deliberately small subset: alignment, zero padding, minimum width, and the +``d`` / ``s`` types. There are no conversions, attribute traversal, indexing, or expressions. For a +computation the blocks cannot express, pass a trusted Python :class:`NameRenderer` for that entry. +""" + +from __future__ import annotations + +import re +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, InstanceOf, JsonValue, model_validator +from pydantic import Field as PydanticField + +from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue + +NAMING_ENTRIES: tuple[str, ...] = ( + "inventory_label", + "inventory_description", + "device_label", + "device_description", + "endpoint_label", + "endpoint_description", +) + +INVENTORY_NAMING_ENTRIES: frozenset[str] = frozenset({"inventory_label", "inventory_description"}) +TOPOLOGY_NAMING_ENTRIES: frozenset[str] = frozenset(NAMING_ENTRIES) - INVENTORY_NAMING_ENTRIES + + +class Text(BaseModel): + """A literal text block: ``Text("-")`` or YAML ``{text: "-"}``.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + text: str + + def __init__(self, text: str | None = None, /, **data: Any) -> None: + if text is not None: + data["text"] = text + super().__init__(**data) + + +class Field(BaseModel): + """One naming fact: ``Field("endpoint.index", format="02d")`` or YAML ``{field: endpoint.index, format: 02d}``. + + ``mapping`` translates the (stringified) value; ``prefix`` / ``suffix`` are only emitted when the + value is present, so an absent optional field leaves no dangling separator text. + """ + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True, validate_by_name=True, validate_by_alias=True) + + path: str = PydanticField(alias="field") + format: str | None = None + mapping: dict[str, str] | None = None + prefix: str = "" + suffix: str = "" + + def __init__(self, path: str | None = None, /, **data: Any) -> None: + if path is not None: + data["path"] = path + super().__init__(**data) + + +class Join(BaseModel): + """Join rendered parts with ``separator``. With ``skip_missing``, parts whose *optional* value is + absent are omitted; unknown fields, type errors, and failed mappings are still errors.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True, validate_by_name=True, validate_by_alias=True) + + parts: list[NameExpr] = PydanticField(alias="join") + separator: str = "" + skip_missing: bool = False + + def __init__(self, parts: list[Any] | None = None, /, **data: Any) -> None: + if parts is not None: + data["parts"] = parts + super().__init__(**data) + + +NameExpr = str | Text | Field | Join +Join.model_rebuild() + + +class NameContext(BaseModel): + """The facts available to naming. Namespaces: ``device`` (key, label, description), ``module`` + (position), ``endpoint`` (direction, media, index, engine, program, leg), ``vertex`` + (factory_label, id), and ``attributes`` (caller-owned scalar facts).""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + device: dict[str, Any] = PydanticField(default_factory=dict) + module: dict[str, Any] = PydanticField(default_factory=dict) + endpoint: dict[str, Any] = PydanticField(default_factory=dict) + vertex: dict[str, Any] = PydanticField(default_factory=dict) + attributes: dict[str, JsonValue] = PydanticField(default_factory=dict) + + def value(self, path: str) -> Any: + """The value at ``path`` (e.g. ``"device.label"``), or ``None`` when absent.""" + namespace, _, rest = path.partition(".") + node: Any = getattr(self, namespace, None) if namespace in _NAMESPACES else None + for part in rest.split(".") if rest else []: + if not isinstance(node, dict): + return None + node = node.get(part) + return node + + +@runtime_checkable +class NameRenderer(Protocol): + """Trusted Python escape hatch for one naming entry; never loaded from YAML.""" + + def render(self, context: NameContext) -> str: ... + + +NamingEntry = NameExpr | InstanceOf[NameRenderer] + + +class NamingScheme(BaseModel): + """Naming entries. Only explicitly supplied entries take part in layering, so a scheme can + override a single entry; an explicit ``None`` disables (unmanages) that entry. + + Precedence: library defaults < engine defaults < resolved blueprint naming < call-time naming. + An entry is replaced as a whole — expressions are never merged structurally. + """ + + model_config = ConfigDict(extra="forbid", frozen=True, arbitrary_types_allowed=True) + + inventory_label: NamingEntry | None = None + inventory_description: NamingEntry | None = None + device_label: NamingEntry | None = None + device_description: NamingEntry | None = None + endpoint_label: NamingEntry | None = None + endpoint_description: NamingEntry | None = None + + @model_validator(mode="after") + def _validate_expressions(self) -> NamingScheme: + problems = [ + f"{entry}: {message}" + for entry, expr in self.entries().items() + for message in validate_expression(entry, expr) + ] + if problems: + raise ValueError("; ".join(problems)) + return self + + def entries(self) -> dict[str, NamingEntry | None]: + """The explicitly supplied entries (including explicit ``None``).""" + return {name: getattr(self, name) for name in NAMING_ENTRIES if name in self.model_fields_set} + + @classmethod + def layered(cls, *schemes: NamingScheme | None) -> NamingScheme: + """Combine schemes entry by entry; later schemes win for the entries they supply.""" + merged: dict[str, Any] = {} + for scheme in schemes: + if scheme is not None: + merged.update(scheme.entries()) + return cls(**merged) + + +class BlueprintNaming(BaseModel): + """The YAML form of naming entries (no Python renderers).""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + inventory_label: NameExpr | None = None + inventory_description: NameExpr | None = None + device_label: NameExpr | None = None + device_description: NameExpr | None = None + endpoint_label: NameExpr | None = None + endpoint_description: NameExpr | None = None + + @model_validator(mode="after") + def _validate_expressions(self) -> BlueprintNaming: + problems = [ + f"{entry}: {message}" + for entry in NAMING_ENTRIES + if entry in self.model_fields_set + for message in validate_expression(entry, getattr(self, entry)) + ] + if problems: + raise ValueError("; ".join(problems)) + return self + + def to_scheme(self) -> NamingScheme: + return NamingScheme(**{name: getattr(self, name) for name in NAMING_ENTRIES if name in self.model_fields_set}) + + +def validate_expression(entry: str, expr: Any) -> list[str]: + """Static checks for one entry: template syntax, known paths, namespaces, and format specs.""" + if not isinstance(expr, (str, Text, Field, Join)): + return [] # None (disabled) or a trusted Python renderer + allowed = _ENTRY_NAMESPACES.get(entry, _NAMESPACES) + problems: list[str] = [] + try: + fields = _fields_of(expr) + except _TemplateSyntaxError as exc: + return [str(exc)] + for field in fields: + problem = _check_field(field, allowed) + if problem: + problems.append(problem) + return problems + + +def render_name(entry: str, expr: Any, context: NameContext) -> str: + """Render ``expr`` for ``entry``; raises :class:`BlueprintValidationError` on missing required + values, type/format errors, failed mappings, or an empty/control-character result.""" + try: + if isinstance(expr, (str, Text, Field, Join)): + value = _render(expr, context) + else: + value = expr.render(context) + if not isinstance(value, str): + raise _RenderError(f"renderer returned {type(value).__name__}, expected str") + except _Missing as missing: + raise _naming_error(entry, f"required value '{missing.path}' is missing", "naming.missing") from None + except _RenderError as exc: + raise _naming_error(entry, str(exc), "naming.render") from None + if not value.strip(): + raise _naming_error(entry, "rendered an empty name", "naming.empty") + if _CONTROL_CHARS.search(value): + raise _naming_error(entry, "rendered a name containing control characters", "naming.invalid") + return value + + +# --- Internal --- + +_NAMESPACES: frozenset[str] = frozenset({"device", "module", "endpoint", "vertex", "attributes"}) +_KNOWN_PATHS: frozenset[str] = frozenset( + { + "device.key", + "device.label", + "device.description", + "module.position", + "endpoint.direction", + "endpoint.media", + "endpoint.index", + "endpoint.engine", + "endpoint.program", + "endpoint.leg", + "vertex.factory_label", + "vertex.id", + } +) +_NON_ENDPOINT_NAMESPACES = frozenset({"device", "module", "attributes"}) +_ENTRY_NAMESPACES: dict[str, frozenset[str]] = { + "inventory_label": _NON_ENDPOINT_NAMESPACES, + "inventory_description": _NON_ENDPOINT_NAMESPACES, + "device_label": _NON_ENDPOINT_NAMESPACES, + "device_description": _NON_ENDPOINT_NAMESPACES, + "endpoint_label": _NAMESPACES, + "endpoint_description": _NAMESPACES, +} +_PATH_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_-]*)+") +_FORMAT_RE = re.compile(r"[<>^]?0?(?:[1-9][0-9]{0,2})?[ds]?") +_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") + + +class _TemplateSyntaxError(ValueError): + pass + + +class _Missing(Exception): + def __init__(self, path: str) -> None: + self.path = path + + +class _RenderError(Exception): + pass + + +def _naming_error(entry: str, message: str, code: str) -> BlueprintValidationError: + return BlueprintValidationError(ValidationIssue(path=f"naming.{entry}", message=message, code=code)) + + +def _compile_template(template: str) -> list[Text | Field]: + """Compile ``"{a.b}-x{c.d:02d}"`` into literal and field blocks (``{{`` / ``}}`` escape braces).""" + parts: list[Text | Field] = [] + buffer = "" + index = 0 + while index < len(template): + char = template[index] + if char == "{": + if template.startswith("{{", index): + buffer += "{" + index += 2 + continue + end = template.find("}", index) + if end == -1: + raise _TemplateSyntaxError(f"unclosed '{{' in template {template!r}") + token = template[index + 1 : end] + if "{" in token: + raise _TemplateSyntaxError(f"nested braces are not supported in template {template!r}") + path, _, spec = token.partition(":") + if buffer: + parts.append(Text(buffer)) + buffer = "" + parts.append(Field(path, format=spec or None)) + index = end + 1 + elif char == "}": + if template.startswith("}}", index): + buffer += "}" + index += 2 + continue + raise _TemplateSyntaxError(f"single '}}' in template {template!r}; use '}}}}' for a literal brace") + else: + buffer += char + index += 1 + if buffer: + parts.append(Text(buffer)) + return parts + + +def _fields_of(expr: Any) -> list[Field]: + if isinstance(expr, str): + return [part for part in _compile_template(expr) if isinstance(part, Field)] + if isinstance(expr, Field): + return [expr] + if isinstance(expr, Join): + return [field for part in expr.parts for field in _fields_of(part)] + return [] + + +def _check_field(field: Field, allowed: frozenset[str]) -> str | None: + path = field.path + if not _PATH_RE.fullmatch(path): + return f"invalid field reference {path!r} (use dotted names such as 'device.label')" + namespace = path.split(".", 1)[0] + if namespace not in _NAMESPACES or (namespace != "attributes" and path not in _KNOWN_PATHS): + return f"unknown field {path!r}" + if namespace not in allowed: + return f"field {path!r} is not available for this entry" + if field.format is not None and not _FORMAT_RE.fullmatch(field.format): + return f"unsupported format {field.format!r} for {path!r} (supported: alignment, 0-padding, width, d, s)" + return None + + +def _render(expr: Any, context: NameContext) -> str: + if isinstance(expr, str): + return "".join(_render(part, context) for part in _compile_template(expr)) + if isinstance(expr, Text): + return expr.text + if isinstance(expr, Field): + return _render_field(expr, context) + if isinstance(expr, Join): + rendered: list[str] = [] + for part in expr.parts: + try: + rendered.append(_render(part, context)) + except _Missing: + if not expr.skip_missing: + raise + return expr.separator.join(rendered) + raise _RenderError(f"unsupported naming block {type(expr).__name__}") + + +def _render_field(field: Field, context: NameContext) -> str: + value = context.value(field.path) + if value is None: + raise _Missing(field.path) + if isinstance(value, (dict, list)): + raise _RenderError(f"'{field.path}' is not a scalar value") + if field.mapping is not None: + key = value if isinstance(value, str) else str(value).lower() if isinstance(value, bool) else str(value) + if key not in field.mapping: + raise _RenderError(f"no mapping for value {key!r} of '{field.path}'") + value = field.mapping[key] + text = _format_value(field, value) + return f"{field.prefix}{text}{field.suffix}" + + +def _format_value(field: Field, value: Any) -> str: + spec = field.format or "" + if spec.endswith("d") and (isinstance(value, bool) or not isinstance(value, int)): + raise _RenderError(f"format {spec!r} requires an integer for '{field.path}'") + if spec.endswith("s") and not isinstance(value, str): + raise _RenderError(f"format {spec!r} requires a string for '{field.path}'") + if isinstance(value, bool): + value = str(value).lower() + if not spec: + return str(value) + return format(value if isinstance(value, (int, float)) else str(value), spec) + + +# Defined after the helpers above, which its construction-time validation uses. +DEFAULT_NAMING = NamingScheme( + inventory_label="{device.label}", + device_label="{device.label}", + endpoint_label=Join( + parts=[ + "{device.label}", + Field("module.position", prefix="M"), + "{endpoint.direction}", + "{endpoint.media}", + Field("endpoint.index", format="02d"), + ], + separator="-", + skip_missing=True, + ), +) +"""Library defaults: Inventory and device labels use ``device.label``; endpoint labels join the device +label, an optional ``M`` component, direction, media, and a two-digit index (for example +``device-a-TX-video-01`` or ``device-a-M1-RX-audio-01``). Descriptions are unmanaged by default.""" + + +__all__ = [ + "DEFAULT_NAMING", + "NAMING_ENTRIES", + "BlueprintNaming", + "Field", + "Join", + "NameContext", + "NameExpr", + "NameRenderer", + "NamingScheme", + "Text", + "render_name", + "validate_expression", +] diff --git a/src/videoipath_automation_tool/blueprints/processors/__init__.py b/src/videoipath_automation_tool/blueprints/processors/__init__.py new file mode 100644 index 0000000..fe2092b --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/processors/__init__.py @@ -0,0 +1,297 @@ +"""Vertex processor contract: one class, one required method, explicit per-engine registration. + +A processor *decides* configuration; it never writes. It receives a detached, read-only +:class:`ProcessingContext` (exact device/module scope, immutable records built from fresh server +reads — no reference to the live Inspect snapshot) and validated parameters, and returns a typed +:class:`~videoipath_automation_tool.blueprints.models.ProcessorResult`. The engine validates the +output (scope, conflicts, capabilities), renders names, compares, and performs the writes. + +Custom processors are trusted Python code, not sandboxed: the contract prevents accidental +library-side writes and enforces scope, but cannot stop a plugin that deliberately opens its own +network connection. +""" + +from __future__ import annotations + +import copy +import re +from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Any, ClassVar, Generic, Literal, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError + +from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ProcessorInputError, ValidationIssue +from videoipath_automation_tool.blueprints.models import InterfaceBinding, ProcessorResult, issues_from_pydantic + +ParamsT = TypeVar("ParamsT", bound=BaseModel) + + +class _Record(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class VertexRecord(_Record): + """Detached view of one vertex: identity, owning port/module, recognition facts, current values.""" + + id: str + port_id: str | None = None + module_id: str | None = None + factory_label: str | None = None + """Device-reported (factory) label of the owning port.""" + port_label: str | None = None + vertex_type: str | None = None + """Direction from ``vertexInfo``: ``In`` / ``Out`` / ``Internal`` / ``Undecided``.""" + kind: str | None = None + """``typeFields.type``: ``codec`` / ``ip`` / ``router`` / ``generic``.""" + label: str = "" + description: str = "" + use_as_endpoint: bool | None = None + active: bool | None = None + sips_mode: str | None = None + tags: tuple[str, ...] = () + codec_format: str | None = None + """Codec ``typeFields.generic.codecFormat`` (``Video`` / ``Audio`` / …).""" + media_type: str | None = None + """Codec ``typeFields.specific.type`` (e.g. ``video`` / ``audio``).""" + sdp_support: bool | None = None + main_destination_port: int | None = None + """Codec ``typeFields.generic.mainDstInfo.port``.""" + spare_destination_port: int | None = None + """Codec ``typeFields.generic.spareDstInfo.port``.""" + supports_static_igmp: bool | None = None + """IP ``typeFields.supportsStaticIgmpCfg``.""" + + +class PortRecord(_Record): + id: str + module_id: str | None = None + factory_label: str | None = None + label: str | None = None + vertex_ids: tuple[str, ...] = () + + +class ModuleRecord(_Record): + id: str + label: str | None = None + local_tags: tuple[str, ...] | None = None + """Locally assigned tags; ``None`` when the server representation cannot distinguish them.""" + + +class DeviceRecord(_Record): + id: str + label: str = "" + """Persisted descriptor label (``""`` when the factory label is in effect).""" + description: str = "" + factory_label: str | None = None + icon_type: str | None = None + icon_size: str | None = None + sdp_strategy: str | None = None + site_id: str | None = None + coordinates: tuple[float, float] | None = None + tags: tuple[str, ...] = () + + +class SourceFacts(_Record): + """Caller facts (no credentials).""" + + key: str + label: str + description: str | None = None + module_position: str | None = None + attributes: dict[str, JsonValue] = Field(default_factory=dict) + + +class DriverContext(_Record): + """An Inventory record's driver identity and custom settings (Python field names).""" + + inventory_id: str + driver_id: str + address: str | None = None + """Management address of the record (``cinfo.address``).""" + custom_settings: dict[str, Any] = Field(default_factory=dict) + + +class ProcessingContext(_Record): + """Everything a processor may use to decide configuration, scoped to one device or module. + + ``vertices`` contains only in-scope vertices (for a module target: that module's vertices); a + processor cannot widen scope by returning other ids. ``owner`` is the topology owner's Inventory + context; ``inventory`` is the source entity's own record when it differs from the owner. + """ + + source: SourceFacts + scope: Literal["device", "module"] + device_id: str + module_id: str | None = None + device: DeviceRecord | None = None + modules: tuple[ModuleRecord, ...] = () + ports: tuple[PortRecord, ...] = () + vertices: tuple[VertexRecord, ...] = () + interfaces: tuple[InterfaceBinding, ...] = () + owner: DriverContext | None = None + inventory: DriverContext | None = None + capabilities: frozenset[str] = frozenset() + + def vertex(self, vertex_id: str) -> VertexRecord | None: + return next((vertex for vertex in self.vertices if vertex.id == vertex_id), None) + + def find_vertices( + self, + *, + kind: str | None = None, + vertex_type: str | None = None, + factory_label: str | None = None, + module_id: str | None = None, + ) -> list[VertexRecord]: + """In-scope vertices matching every given filter (exact comparisons), ordered by id.""" + return [ + vertex + for vertex in sorted(self.vertices, key=lambda record: record.id) + if (kind is None or vertex.kind == kind) + and (vertex_type is None or vertex.vertex_type == vertex_type) + and (factory_label is None or vertex.factory_label == factory_label) + and (module_id is None or vertex.module_id == module_id) + ] + + def require_vertex( + self, + *, + kind: str | None = None, + vertex_type: str | None = None, + factory_label: str | None = None, + module_id: str | None = None, + ) -> VertexRecord: + """Exactly one matching in-scope vertex; raises :class:`ProcessorInputError` otherwise.""" + matches = self.find_vertices( + kind=kind, vertex_type=vertex_type, factory_label=factory_label, module_id=module_id + ) + if len(matches) != 1: + criteria = ", ".join( + f"{name}={value!r}" + for name, value in ( + ("kind", kind), + ("vertex_type", vertex_type), + ("factory_label", factory_label), + ("module_id", module_id), + ) + if value is not None + ) + found = "no vertex" if not matches else f"{len(matches)} vertices ({', '.join(v.id for v in matches)})" + raise ProcessorInputError( + f"Expected exactly one vertex with {criteria} in {self.scope_label}; found {found}." + ) + return matches[0] + + @property + def scope_label(self) -> str: + return ( + f"module '{self.module_id}' of device '{self.device_id}'" + if self.module_id + else f"device '{self.device_id}'" + ) + + +class VertexProcessor(ABC, Generic[ParamsT]): + """Base class for processors. Declare ``params_model`` and implement :meth:`process`. + + A new instance is created for every processing run, so instances may keep run-local state. + """ + + params_model: ClassVar[type[BaseModel]] + + @abstractmethod + def process(self, context: ProcessingContext, params: ParamsT) -> ProcessorResult: + """Interpret the scoped topology and propose configuration. Raise + :class:`ProcessorInputError` when the observed topology is unsupported or incomplete.""" + + +class ProcessorRegistry: + """Per-engine processor registrations (never process-global). Built-ins are included by default; + duplicate ids fail — a built-in is never silently overridden.""" + + def __init__( + self, + processors: Mapping[str, type[VertexProcessor[Any]]] | None = None, + *, + include_builtins: bool = True, + ) -> None: + self._processors: dict[str, type[VertexProcessor[Any]]] = {} + if include_builtins: + for processor_id, processor_cls in _builtin_processors().items(): + self.register(processor_id, processor_cls) + for processor_id, processor_cls in dict(processors or {}).items(): + self.register(processor_id, processor_cls) + + def register(self, processor_id: str, processor_cls: type[VertexProcessor[Any]]) -> None: + if not isinstance(processor_id, str) or not _PROCESSOR_ID_RE.fullmatch(processor_id): + raise ValueError(f"Invalid processor id {processor_id!r} (letters, digits, '.', '_', '-').") + if not (isinstance(processor_cls, type) and issubclass(processor_cls, VertexProcessor)): + raise TypeError(f"Processor '{processor_id}' must be a VertexProcessor subclass.") + params_model = getattr(processor_cls, "params_model", None) + if not (isinstance(params_model, type) and issubclass(params_model, BaseModel)): + raise TypeError(f"Processor '{processor_id}' must declare 'params_model' as a pydantic model class.") + if processor_id in self._processors: + raise ValueError(f"Processor id '{processor_id}' is already registered; use a new id.") + self._processors[processor_id] = processor_cls + + def get(self, processor_id: str) -> type[VertexProcessor[Any]]: + processor_cls = self._processors.get(processor_id) + if processor_cls is None: + raise BlueprintValidationError( + ValidationIssue( + message=f"Unknown processor '{processor_id}'. Available: {', '.join(self.ids()) or 'none'}.", + code="processor.unknown", + ) + ) + return processor_cls + + def ids(self) -> list[str]: + return sorted(self._processors) + + def __contains__(self, processor_id: object) -> bool: + return processor_id in self._processors + + def copy(self) -> ProcessorRegistry: + clone = ProcessorRegistry(include_builtins=False) + clone._processors = dict(self._processors) + return clone + + def params_schema(self, processor_id: str) -> dict[str, Any]: + return self.get(processor_id).params_model.model_json_schema() + + def validate_params(self, processor_id: str, params: Mapping[str, Any], path: tuple[Any, ...] = ()) -> BaseModel: + """Validate ``params`` with the processor's model; issues carry ``path`` (document location).""" + params_model = self.get(processor_id).params_model + data = copy.deepcopy(dict(params)) + try: + return params_model.model_validate(data) + except ValidationError as exc: + raise BlueprintValidationError(issues_from_pydantic(exc, data=data, prefix=path)) from None + + def __repr__(self) -> str: + return f"ProcessorRegistry({', '.join(self.ids())})" + + +# --- Internal --- + +_PROCESSOR_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") + + +def _builtin_processors() -> dict[str, type[VertexProcessor[Any]]]: + from videoipath_automation_tool.blueprints.processors.matrox import MatroxConvertIPProcessor + + return {"matrox.convertip.default": MatroxConvertIPProcessor} + + +__all__ = [ + "DeviceRecord", + "DriverContext", + "ModuleRecord", + "PortRecord", + "ProcessingContext", + "ProcessorRegistry", + "SourceFacts", + "VertexProcessor", + "VertexRecord", +] diff --git a/src/videoipath_automation_tool/blueprints/processors/matrox.py b/src/videoipath_automation_tool/blueprints/processors/matrox.py new file mode 100644 index 0000000..b6d60a9 --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/processors/matrox.py @@ -0,0 +1,263 @@ +"""Built-in processor for Matrox ConvertIP devices (``matrox.convertip.default``). + +Supported layouts (recognized from codec composition and direction, never from a vertex count alone): + +- ``tx``: one video and one audio transmitter. +- ``rx``: one video and one audio receiver. +- ``four_split``: four video receivers with distinct indices 1–4 (parsed from the factory label + ``Video Receiver ``, zero-based on the device) and one audio receiver. + +For the observed codec convention an ``In`` codec vertex is a transmitter and an ``Out`` codec vertex +a receiver; this interpretation is local to this processor. Media is taken from structured codec +information first (``codecFormat`` / specific type) and only then from ``Video`` / ``Audio`` words in +the factory label. ``mode`` asserts the recognized layout; it does not switch hardware between TX and +RX (that is a device/driver change followed by synchronization and replanning). + +Proposed edits: endpoint enablement, semantic endpoint identity (engine ``0``, program ``1``), the +factory label as description (joined with the caller's device description when given), sender SDP +support, SIPS mode and category tags only when requested, and a ``gateway`` (TX) / ``monitor`` (RX, +four-split) icon as a device-target default. Vertex control is not configured: its Inspect mapping +is unverified, so ``configure_control: true`` fails with a capability error. +""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict + +from videoipath_automation_tool.blueprints.errors import BlueprintCapabilityError, ProcessorInputError +from videoipath_automation_tool.blueprints.models import ( + DevicePatch, + Diagnostic, + EndpointIdentity, + ProcessorResult, + TagSpec, + VertexEdit, + VertexPatch, +) +from videoipath_automation_tool.blueprints.processors import ProcessingContext, VertexProcessor, VertexRecord + +Layout = Literal["tx", "rx", "four_split"] + + +class MatroxConvertIPParams(BaseModel): + """Parameters. Omitted tag parameters leave local tags unchanged (``[]`` clears them); an omitted + ``redundant_streams`` preserves the SIPS mode (``true`` → ``SIPSAuto``, ``false`` → ``NONE``).""" + + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + + mode: Literal["auto", "tx", "rx", "four_split"] = "auto" + video_receiver_tags: TagSpec | None = None + video_sender_tags: TagSpec | None = None + audio_receiver_tags: TagSpec | None = None + audio_sender_tags: TagSpec | None = None + redundant_streams: bool | None = None + allow_null_audio_label: bool = False + """Compatibility for a driver defect that labels the single audio endpoint ``null``.""" + configure_control: bool = False + + +class MatroxConvertIPProcessor(VertexProcessor[MatroxConvertIPParams]): + params_model = MatroxConvertIPParams + + def process(self, context: ProcessingContext, params: MatroxConvertIPParams) -> ProcessorResult: + if params.configure_control: + raise BlueprintCapabilityError( + "Matrox ConvertIP: configuring vertex control is not supported until its Inspect mapping is verified; " + "remove 'configure_control' to preserve the current control settings." + ) + diagnostics: list[Diagnostic] = [] + + unknown = [vertex.id for vertex in context.vertices if vertex.kind is None] + if unknown: + raise ProcessorInputError( + f"Matrox ConvertIP: unknown kind for vertices {', '.join(sorted(unknown))} in {context.scope_label} " + "(no edit form; incomplete discovery?)." + ) + codecs = context.find_vertices(kind="codec") + if not codecs: + raise ProcessorInputError( + f"Matrox ConvertIP: no codec vertices in {context.scope_label} (incomplete discovery or unsupported layout)." + ) + media = self._classify_media(codecs, params, diagnostics) + direction = self._direction(codecs) + layout = self._layout(direction, codecs, media) + if params.mode != "auto" and params.mode != layout: + raise ProcessorInputError( + f"Matrox ConvertIP: mode '{params.mode}' was requested but {context.scope_label} has a '{layout}' layout." + ) + indices = self._indices(layout, codecs, media) + + if params.redundant_streams: + diagnostics.append(self._check_redundancy(context, codecs[0].vertex_type or "")) + + edits = [ + VertexEdit( + vertex_id=vertex.id, + fields=self._fields(vertex, direction, media[vertex.id], params, context), + endpoint=EndpointIdentity( + direction=direction, media=media[vertex.id], index=indices[vertex.id], engine=0, program=1 + ), + ) + for vertex in codecs + ] + device = ( + DevicePatch(icon_type="gateway" if layout == "tx" else "monitor") if context.scope == "device" else None + ) + diagnostics.append(Diagnostic(code="matrox.layout", message=f"Recognized '{layout}' layout.")) + return ProcessorResult(vertices=edits, device=device, diagnostics=diagnostics) + + # --- Recognition --- + + @staticmethod + def _classify_media( + codecs: list[VertexRecord], params: MatroxConvertIPParams, diagnostics: list[Diagnostic] + ) -> dict[str, str]: + media: dict[str, str] = {} + unknown: list[VertexRecord] = [] + for vertex in codecs: + kind = _structured_media(vertex) or _label_media(vertex.factory_label) + if kind is None: + unknown.append(vertex) + else: + media[vertex.id] = kind + + if unknown and params.allow_null_audio_label: + null_labelled = [v for v in unknown if (v.factory_label or "").strip().lower() == "null"] + if len(unknown) == 1 and len(null_labelled) == 1 and "audio" not in media.values(): + vertex = null_labelled[0] + media[vertex.id] = "audio" + unknown = [] + diagnostics.append( + Diagnostic( + level="warning", + code="matrox.null_audio_label", + message="Treated the single codec vertex labelled 'null' as the audio endpoint " + "(allow_null_audio_label compatibility workaround).", + entity_id=vertex.id, + ) + ) + if unknown: + labels = ", ".join(f"{v.id} ({v.factory_label!r})" for v in unknown) + raise ProcessorInputError(f"Matrox ConvertIP: cannot classify codec vertices as video or audio: {labels}.") + return media + + @staticmethod + def _direction(codecs: list[VertexRecord]) -> str: + directions = {vertex.vertex_type for vertex in codecs} + if len(directions) != 1 or next(iter(directions)) not in _DIRECTIONS: + raise ProcessorInputError( + f"Matrox ConvertIP: codec vertices must all be 'In' (TX) or all 'Out' (RX); found {sorted(map(str, directions))}." + ) + return _DIRECTIONS[next(iter(directions))] + + @staticmethod + def _layout(direction: str, codecs: list[VertexRecord], media: dict[str, str]) -> Layout: + videos = sum(1 for vertex in codecs if media[vertex.id] == "video") + audios = sum(1 for vertex in codecs if media[vertex.id] == "audio") + if (videos, audios) == (1, 1): + return "tx" if direction == "TX" else "rx" + if direction == "RX" and (videos, audios) == (4, 1): + return "four_split" + raise ProcessorInputError( + f"Matrox ConvertIP: unsupported {direction} layout with {videos} video and {audios} audio codec vertices." + ) + + @staticmethod + def _indices(layout: Layout, codecs: list[VertexRecord], media: dict[str, str]) -> dict[str, int]: + indices = {vertex.id: 1 for vertex in codecs} + if layout != "four_split": + return indices + for vertex in codecs: + if media[vertex.id] != "video": + continue + match = _FOUR_SPLIT_INDEX.search((vertex.factory_label or "").strip()) + if match is None: + raise ProcessorInputError( + f"Matrox ConvertIP: four-split video vertex {vertex.id} has no 'Video Receiver ' index " + f"in its factory label {vertex.factory_label!r}." + ) + indices[vertex.id] = int(match.group(1)) + 1 + found = sorted(indices[v.id] for v in codecs if media[v.id] == "video") + if found != [1, 2, 3, 4]: + raise ProcessorInputError( + f"Matrox ConvertIP: four-split video indices must be 0–3 on the device (1–4 after normalization); found {found}." + ) + return indices + + @staticmethod + def _check_redundancy(context: ProcessingContext, codec_type: str) -> Diagnostic: + stream_type = "Out" if codec_type == "In" else "In" + if context.interfaces: + streaming = [ + vertex_id + for binding in context.interfaces + if (vertex_id := binding.out_vertex_id if stream_type == "Out" else binding.in_vertex_id) is not None + ] + origin = "ip_vertex_mapping" + else: + streaming = [vertex.id for vertex in context.find_vertices(kind="ip", vertex_type=stream_type)] + origin = "in-scope IP vertices" + if len(streaming) != 2: + raise ProcessorInputError( + f"Matrox ConvertIP: redundant_streams requires exactly two '{stream_type}' streaming IP vertices " + f"({origin}); found {len(streaming)}. Use ip_vertex_mapping to select the streaming pair." + ) + return Diagnostic( + code="matrox.redundancy", + message=f"Redundant streaming pair {', '.join(sorted(streaming))} ({origin}); " + "stream paths through the pair are not verified by this processor.", + ) + + # --- Proposals --- + + @staticmethod + def _fields( + vertex: VertexRecord, direction: str, media: str, params: MatroxConvertIPParams, context: ProcessingContext + ) -> VertexPatch: + description = vertex.factory_label or "" + if context.source.description: + description = f"{description} | {context.source.description}" if description else context.source.description + fields: dict[str, object] = {"use_as_endpoint": True, "description": description} + if direction == "TX": + fields["sdp_support"] = True + if params.redundant_streams is not None: + fields["sips_mode"] = "SIPSAuto" if params.redundant_streams else "NONE" + tags = { + ("video", "TX"): params.video_sender_tags, + ("video", "RX"): params.video_receiver_tags, + ("audio", "TX"): params.audio_sender_tags, + ("audio", "RX"): params.audio_receiver_tags, + }[(media, direction)] + if tags is not None: + fields["tags"] = tags + return VertexPatch(**fields) + + +# --- Internal --- + +_DIRECTIONS = {"In": "TX", "Out": "RX"} +_FOUR_SPLIT_INDEX = re.compile(r"Video Receiver\s+(\d+)$", re.IGNORECASE) +_VIDEO_WORD = re.compile(r"\bvideo\b", re.IGNORECASE) +_AUDIO_WORD = re.compile(r"\baudio\b", re.IGNORECASE) + + +def _structured_media(vertex: VertexRecord) -> str | None: + for value in (vertex.codec_format, vertex.media_type): + if value and value.lower() in ("video", "audio"): + return value.lower() + return None + + +def _label_media(label: str | None) -> str | None: + if not label: + return None + is_video, is_audio = bool(_VIDEO_WORD.search(label)), bool(_AUDIO_WORD.search(label)) + if is_video == is_audio: + return None + return "video" if is_video else "audio" + + +__all__ = ["MatroxConvertIPParams", "MatroxConvertIPProcessor"] diff --git a/src/videoipath_automation_tool/blueprints/resolution.py b/src/videoipath_automation_tool/blueprints/resolution.py new file mode 100644 index 0000000..8c9c0db --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/resolution.py @@ -0,0 +1,509 @@ +"""Variant resolution, driver/parameter validation, and JSON Schema export. + +Resolution order per section: the ``default`` entry, then the selected named entry (a patch), then +per-instance Inventory overrides. Merge rules are small and fixed: + +- mappings merge recursively by key; scalars replace; lists replace entirely; ``{}`` adds nothing; +- naming expressions and tag specifications are replaced as a whole; +- ``vertex_processor: null`` disables an inherited processor; a different ``processor_type`` + replaces the whole processor specification (parameters are not inherited); +- a different ``driver_id`` replaces the inherited ``custom_settings``. + +The source document is never mutated. Every merged leaf records its provenance (``default``, +``variant:``, ``instance``) so plans can explain where a managed value came from. +""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.resources +import json +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, ValidationError + +from videoipath_automation_tool.apps.inventory.model.drivers import ( + DRIVER_ID_TO_CUSTOM_SETTINGS, + SELECTED_SCHEMA_VERSION, +) +from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue +from videoipath_automation_tool.blueprints.models import ( + Blueprint, + InventoryConfig, + InventorySettings, + Scope, + TopologyConfig, + issues_from_pydantic, +) +from videoipath_automation_tool.blueprints.naming import NamingScheme +from videoipath_automation_tool.blueprints.processors import ProcessorRegistry, VertexProcessor + +Provenance = dict[str, str] + + +class ResolvedInventory(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + variant: str + driver_id: str + schema_version: str = SELECTED_SCHEMA_VERSION + config: InventoryConfig + custom_settings: dict[str, Any] + """Explicitly configured custom settings (Python field names), validated against the driver model.""" + provenance: Provenance + + +class ResolvedTopology(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + variant: str + config: TopologyConfig + processor_id: str | None = None + processor_cls: type[VertexProcessor] | None = None # type: ignore[type-arg] + params: BaseModel | None = None + provenance: Provenance + + +class ResolvedBlueprint(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + inventory: ResolvedInventory | None = None + topology: ResolvedTopology | None = None + naming: NamingScheme + skipped: dict[str, str] + digest: str + + +def resolve_blueprint( + blueprint: Blueprint, + *, + scope: Scope, + inventory_variant: str, + topology_variant: str, + registry: ProcessorRegistry, + overrides: InventorySettings | None = None, +) -> ResolvedBlueprint: + """Resolve the selected sections and variants; raises :class:`BlueprintValidationError`.""" + if scope not in ("all", "inventory", "topology"): + raise BlueprintValidationError(f"Invalid scope {scope!r}; expected 'all', 'inventory', or 'topology'.") + selected = { + "inventory": _select(blueprint, "inventory", inventory_variant, scope), + "topology": _select(blueprint, "topology", topology_variant, scope), + } + skipped = {section: reason for section, reason in selected.items() if reason is not None} + + inventory = None + if "inventory" not in skipped: + inventory = resolve_inventory(blueprint, inventory_variant, overrides=overrides) + topology = None + if "topology" not in skipped: + topology = resolve_topology(blueprint, topology_variant, registry=registry, require_processor=True) + + naming = NamingScheme.layered( + inventory.config.naming.to_scheme() if inventory and inventory.config.naming else None, + topology.config.naming.to_scheme() if topology and topology.config.naming else None, + ) + return ResolvedBlueprint( + inventory=inventory, topology=topology, naming=naming, skipped=skipped, digest=blueprint_digest(blueprint) + ) + + +def resolve_inventory( + blueprint: Blueprint, variant: str, *, overrides: InventorySettings | None = None +) -> ResolvedInventory: + raw = _raw_entries(blueprint, "inventory") + base = copy.deepcopy(raw["default"]) + provenance = _leaf_provenance(base, "default") + if variant != "default": + patch = raw[variant] + if "driver_id" in patch and patch["driver_id"] != base.get("driver_id"): + base.pop("custom_settings", None) + provenance = {path: src for path, src in provenance.items() if not path.startswith("custom_settings")} + base = _merge(base, patch, (), provenance, f"variant:{variant}") + if overrides is not None: + base = _merge(base, overrides.model_dump(exclude_unset=True), (), provenance, "instance") + + entry_path = ("inventory", variant) + config = _validate_entry(InventoryConfig, base, blueprint, entry_path, provenance) + if config.driver_id is None: + raise BlueprintValidationError( + _issue(blueprint, entry_path + ("driver_id",), "Field required (inventory needs a driver_id).", "missing") + ) + model = DRIVER_ID_TO_CUSTOM_SETTINGS.get(config.driver_id) + if model is None: + raise BlueprintValidationError( + _issue( + blueprint, + _source_path(entry_path, ("driver_id",), provenance, "driver_id"), + f"Unknown driver '{config.driver_id}' for the selected driver schema {SELECTED_SCHEMA_VERSION}.", + "driver.unknown", + ) + ) + custom = dict(config.custom_settings or {}) + return ResolvedInventory( + variant=variant, + driver_id=config.driver_id, + config=config, + custom_settings=_validate_custom_settings(model, custom, blueprint, entry_path, provenance, config.driver_id), + provenance=provenance, + ) + + +def resolve_topology( + blueprint: Blueprint, + variant: str, + *, + registry: ProcessorRegistry | None, + require_processor: bool, +) -> ResolvedTopology: + raw = _raw_entries(blueprint, "topology") + base = copy.deepcopy(raw["default"]) + provenance = _leaf_provenance(base, "default") + if variant != "default": + patch = copy.deepcopy(raw[variant]) + source = f"variant:{variant}" + if "vertex_processor" in patch: + base["vertex_processor"] = _merge_processor( + base.get("vertex_processor"), patch.pop("vertex_processor"), provenance, source + ) + base = _merge(base, patch, (), provenance, source) + + entry_path = ("topology", variant) + config = _validate_entry(TopologyConfig, base, blueprint, entry_path, provenance) + spec = config.vertex_processor + if spec is None: + return ResolvedTopology(variant=variant, config=config, provenance=provenance) + if spec.processor_type is None: + raise BlueprintValidationError( + _issue(blueprint, entry_path + ("vertex_processor", "processor_type"), "Field required.", "missing") + ) + if registry is None or (spec.processor_type not in registry and not require_processor): + return ResolvedTopology(variant=variant, config=config, processor_id=spec.processor_type, provenance=provenance) + + type_path = _source_path( + entry_path, ("vertex_processor", "processor_type"), provenance, "vertex_processor.processor_type" + ) + try: + processor_cls = registry.get(spec.processor_type) + except BlueprintValidationError as exc: + raise BlueprintValidationError( + _issue(blueprint, type_path, exc.issues[0].message, "processor.unknown") + ) from None + try: + params = registry.validate_params(spec.processor_type, spec.params) + except BlueprintValidationError as exc: + raise BlueprintValidationError( + [ + _relocate_relative( + blueprint, + entry_path, + issue.model_copy(update={"path": f"vertex_processor.params.{issue.path}".rstrip(".")}), + provenance, + ) + for issue in exc.issues + ] + ) from None + return ResolvedTopology( + variant=variant, + config=config, + processor_id=spec.processor_type, + processor_cls=processor_cls, + params=params, + provenance=provenance, + ) + + +def validate_all_variants(blueprint: Blueprint, registry: ProcessorRegistry | None) -> None: + """Resolve every variant of every section; collect and raise all issues.""" + issues: list[ValidationIssue] = [] + for variant in blueprint.variant_names("inventory"): + try: + resolve_inventory(blueprint, variant) + except BlueprintValidationError as exc: + issues.extend(exc.issues) + for variant in blueprint.variant_names("topology"): + try: + resolved = resolve_topology(blueprint, variant, registry=registry, require_processor=False) + except BlueprintValidationError as exc: + issues.extend(exc.issues) + continue + if registry is not None and resolved.processor_id is not None and resolved.processor_cls is None: + issues.append( + _issue( + blueprint, + ("topology", variant, "vertex_processor", "processor_type"), + f"Processor '{resolved.processor_id}' is not registered; its parameters were not validated.", + "processor.unavailable", + ) + ) + if issues: + raise BlueprintValidationError(issues) + + +def blueprint_digest(blueprint: Blueprint) -> str: + """Stable digest of the normalized document.""" + document = blueprint.model_dump(mode="json", by_alias=True, exclude_unset=True) + return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def blueprint_json_schema( + registry: ProcessorRegistry | None = None, *, restrict_processor_types: bool = False +) -> dict[str, Any]: + """JSON Schema (draft 2020-12) generated from the runtime models. With ``registry``, the + ``params`` of each registered ``processor_type`` are described through ``if``/``then`` branches. + + ``restrict_processor_types`` (requires ``registry``) limits ``processor_type`` to the registered + ids, so editors flag unknown or misspelled processors. A variant may still omit it to inherit + the ``default`` processor. Without it, unknown ids are accepted and their ``params`` are + unconstrained (the packaged schema stays usable with processors registered elsewhere). + """ + if restrict_processor_types and registry is None: + raise ValueError("restrict_processor_types requires a registry.") + schema = Blueprint.model_json_schema(by_alias=True) + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", + **schema, + "title": "VideoIPath Automation Tool blueprint (schema_version 1)", + } + if registry is None: + return schema + defs = schema.setdefault("$defs", {}) + branches: list[dict[str, Any]] = [] + for processor_id in registry.ids(): + params_schema = registry.get(processor_id).params_model.model_json_schema( + by_alias=True, ref_template="#/$defs/{model}" + ) + defs.update(params_schema.pop("$defs", {})) + branches.append( + { + "if": {"properties": {"processor_type": {"const": processor_id}}, "required": ["processor_type"]}, + "then": {"properties": {"params": params_schema}}, + } + ) + spec = defs["VertexProcessorSpec"] + if branches: + spec["allOf"] = branches + if restrict_processor_types: + processor_type = spec["properties"]["processor_type"] + processor_type.pop("anyOf", None) + processor_type.update({"type": "string", "enum": registry.ids()}) + return schema + + +def published_json_schema() -> dict[str, Any]: + """The packaged ``schemas/blueprint-v1.schema.json`` (document plus built-in processor parameters).""" + resource = importlib.resources.files("videoipath_automation_tool.blueprints").joinpath( + "schemas/blueprint-v1.schema.json" + ) + return json.loads(resource.read_text(encoding="utf-8")) + + +# --- Internal --- + +_ATOMIC_KEYS = frozenset({"tags"}) + + +def _select(blueprint: Blueprint, section: Literal["inventory", "topology"], variant: str, scope: Scope) -> str | None: + """``None`` when the section is selected, else the skip reason. Raises for invalid requests.""" + names = blueprint.variant_names(section) + if not names: + if scope == section: + raise BlueprintValidationError( + ValidationIssue( + path=section, + message=f"scope='{section}' requested but the blueprint has no '{section}' section.", + code="section.missing", + ) + ) + if variant != "default": + raise BlueprintValidationError( + ValidationIssue( + path=section, + message=f"variant '{variant}' requested for the absent '{section}' section.", + code="section.missing", + ) + ) + return "section absent from blueprint" + if variant not in names: + raise BlueprintValidationError( + ValidationIssue( + path=section, + message=f"Unknown {section} variant '{variant}'. Available: {', '.join(names)}.", + code="variant.unknown", + ) + ) + if scope not in ("all", section): + return f"not in scope '{scope}'" + return None + + +def _raw_entries(blueprint: Blueprint, section: str) -> dict[str, dict[str, Any]]: + document = blueprint.model_dump(mode="python", by_alias=True, exclude_unset=True) + return document.get(section) or {} + + +def _merge( + base: dict[str, Any], patch: dict[str, Any], path: tuple[str, ...], provenance: Provenance, source: str +) -> dict[str, Any]: + merged = dict(base) + for key, value in patch.items(): + child = path + (key,) + current = merged.get(key) + if isinstance(value, dict) and isinstance(current, dict) and not _atomic(child): + merged[key] = _merge(current, value, child, provenance, source) + continue + merged[key] = copy.deepcopy(value) + prefix = ".".join(child) + for stale in [p for p in provenance if p == prefix or p.startswith(prefix + ".")]: + del provenance[stale] + provenance.update(_leaf_provenance(value, source, child)) + return merged + + +def _merge_processor(base: Any, patch: Any, provenance: Provenance, source: str) -> Any: + """``null`` disables; a different ``processor_type`` replaces the whole spec; else params merge.""" + replaces = ( + patch is None + or not isinstance(base, dict) + or not isinstance(patch, dict) + or patch.get("processor_type", base.get("processor_type")) != base.get("processor_type") + ) + if not replaces: + return _merge(base, patch, ("vertex_processor",), provenance, source) + for stale in [p for p in provenance if p == "vertex_processor" or p.startswith("vertex_processor.")]: + del provenance[stale] + provenance.update(_leaf_provenance(patch, source, ("vertex_processor",))) + return copy.deepcopy(patch) + + +def _atomic(path: tuple[str, ...]) -> bool: + """Naming expressions and tag specifications are replaced as a whole.""" + return (len(path) == 2 and path[0] == "naming") or path[-1] in _ATOMIC_KEYS or path[-1].endswith("_tags") + + +def _leaf_provenance(value: Any, source: str, path: tuple[str, ...] = ()) -> Provenance: + if isinstance(value, dict) and value and not (path and _atomic(path)): + result: Provenance = {} + for key, child in value.items(): + result.update(_leaf_provenance(child, source, path + (str(key),))) + return result + return {".".join(path): source} if path else {} + + +def _source_path( + entry_path: tuple[str, ...], relative: tuple[Any, ...], provenance: Provenance, key: str +) -> tuple[Any, ...]: + """Document path of a merged field: point at the entry the value actually came from.""" + source = provenance.get(key) or next( + (src for path, src in provenance.items() if path.startswith(key + ".") or key.startswith(path + ".")), None + ) + section = entry_path[0] + if source == "default": + return (section, "default") + relative + if source == "instance": + return ("device", "inventory_overrides") + relative + return entry_path + relative + + +def _validate_entry( + model: type[BaseModel], + data: dict[str, Any], + blueprint: Blueprint, + entry_path: tuple[str, ...], + provenance: Provenance, +) -> Any: + try: + return model.model_validate(data) + except ValidationError as exc: + issues = issues_from_pydantic(exc, data=data) + raise BlueprintValidationError( + [_relocate_relative(blueprint, entry_path, issue, provenance) for issue in issues] + ) from None + + +def _validate_custom_settings( + model: type[BaseModel], + custom: dict[str, Any], + blueprint: Blueprint, + entry_path: tuple[str, ...], + provenance: Provenance, + driver_id: str, +) -> dict[str, Any]: + allowed = set(model.model_fields) - {"driver_id"} + unknown = sorted(set(custom) - allowed) + if unknown: + raise BlueprintValidationError( + [ + _issue( + blueprint, + _source_path(entry_path, ("custom_settings", key), provenance, f"custom_settings.{key}"), + f"Unknown custom setting for driver '{driver_id}' (schema {SELECTED_SCHEMA_VERSION}). " + f"Allowed: {', '.join(sorted(allowed)) or 'none'}.", + "driver.unknown_setting", + ) + for key in unknown + ] + ) + try: + instance = model.model_validate(custom, strict=True, by_name=True, by_alias=False) + except ValidationError as exc: + issues = issues_from_pydantic(exc, data=custom) + raise BlueprintValidationError( + [ + _relocate( + blueprint, + _source_path(entry_path, ("custom_settings",), provenance, f"custom_settings.{issue.path}"), + issue.model_copy( + update={"message": f"{issue.message} (driver '{driver_id}', schema {SELECTED_SCHEMA_VERSION})"} + ), + ) + for issue in issues + ] + ) from None + return {key: getattr(instance, key) for key in custom} + + +def _relocate_relative( + blueprint: Blueprint, entry_path: tuple[str, ...], issue: ValidationIssue, provenance: Provenance +) -> ValidationIssue: + relative = tuple(issue.path.split(".")) if issue.path else () + path = _source_path(entry_path, relative, provenance, issue.path) + return _issue(blueprint, path, issue.message, issue.code) + + +def _relocate(blueprint: Blueprint, prefix: tuple[Any, ...], issue: ValidationIssue) -> ValidationIssue: + relative = tuple(issue.path.split(".")) if issue.path else () + return _issue(blueprint, prefix + relative, issue.message, issue.code) + + +def _issue(blueprint: Blueprint, path: tuple[Any, ...], message: str, code: str) -> ValidationIssue: + keys = tuple(int(part) if isinstance(part, str) and part.isdigit() else part for part in path) + source, line, column = None, None, None + for end in range(len(keys), -1, -1): + source, line, column = blueprint.location_of(keys[:end]) + if line is not None: + break + return ValidationIssue( + path=".".join(str(part) for part in path), + message=message, + code=code, + source=source, + line=line, + column=column, + ) + + +__all__ = [ + "ResolvedBlueprint", + "ResolvedInventory", + "ResolvedTopology", + "blueprint_digest", + "blueprint_json_schema", + "published_json_schema", + "resolve_blueprint", + "resolve_inventory", + "resolve_topology", + "validate_all_variants", +] diff --git a/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json b/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json new file mode 100644 index 0000000..97bba6c --- /dev/null +++ b/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json @@ -0,0 +1,1159 @@ +{ + "$defs": { + "BlueprintNaming": { + "additionalProperties": false, + "description": "The YAML form of naming entries (no Python renderers).", + "properties": { + "device_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Device Description" + }, + "device_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Device Label" + }, + "endpoint_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Endpoint Description" + }, + "endpoint_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Endpoint Label" + }, + "inventory_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory Description" + }, + "inventory_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory Label" + } + }, + "title": "BlueprintNaming", + "type": "object" + }, + "CatalogId": { + "additionalProperties": false, + "description": "Exact catalog id reference, e.g. ``{id: default}`` for the system SNMP configuration.", + "properties": { + "id": { + "minLength": 1, + "title": "Id", + "type": "string" + } + }, + "required": [ + "id" + ], + "title": "CatalogId", + "type": "object" + }, + "Coordinates": { + "additionalProperties": false, + "properties": { + "x": { + "title": "X", + "type": "number" + }, + "y": { + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "title": "Coordinates", + "type": "object" + }, + "DevicePatch": { + "additionalProperties": false, + "description": "Inspect device fields (device targets only). Device label/description come from naming.", + "properties": { + "coordinates": { + "anyOf": [ + { + "$ref": "#/$defs/Coordinates" + }, + { + "type": "null" + } + ], + "default": null + }, + "icon_size": { + "anyOf": [ + { + "enum": [ + "auto", + "large", + "medium", + "small" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Icon Size" + }, + "icon_type": { + "anyOf": [ + { + "enum": [ + "default", + "none", + "device", + "camera", + "monitor", + "encoder", + "decoder", + "audioMixer", + "videoMixer", + "processingDevice", + "transportStreamProcessor", + "mediaDevice", + "server", + "gateway", + "ipSwitchRouter", + "vlanCloud", + "videoAudioRouterMatrix", + "encoderDecoder" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Icon Type" + }, + "sdp_strategy": { + "anyOf": [ + { + "enum": [ + "always", + "once", + "video" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sdp Strategy" + }, + "site_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Site Id" + }, + "tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "title": "DevicePatch", + "type": "object" + }, + "Field": { + "additionalProperties": false, + "description": "One naming fact: ``Field(\"endpoint.index\", format=\"02d\")`` or YAML ``{field: endpoint.index, format: 02d}``.\n\n``mapping`` translates the (stringified) value; ``prefix`` / ``suffix`` are only emitted when the\nvalue is present, so an absent optional field leaves no dangling separator text.", + "properties": { + "field": { + "title": "Field", + "type": "string" + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Format" + }, + "mapping": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mapping" + }, + "prefix": { + "default": "", + "title": "Prefix", + "type": "string" + }, + "suffix": { + "default": "", + "title": "Suffix", + "type": "string" + } + }, + "required": [ + "field" + ], + "title": "Field", + "type": "object" + }, + "GenericSettings": { + "additionalProperties": false, + "description": "Generic connection settings (``cinfo.http``). ``http_auth`` is the server's raw\n``httpAuth`` mode code (``0`` disables HTTP authentication).", + "properties": { + "enable_https": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enable Https" + }, + "http_auth": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Http Auth" + }, + "trust_all_certificates": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Trust All Certificates" + } + }, + "title": "GenericSettings", + "type": "object" + }, + "InventoryConfig": { + "additionalProperties": false, + "description": "One Inventory entry (``default`` or a named variant patch).", + "properties": { + "active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Active" + }, + "custom_settings": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Custom Settings" + }, + "driver_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Driver Id" + }, + "generic_settings": { + "anyOf": [ + { + "$ref": "#/$defs/GenericSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "propertyNames": { + "minLength": 1 + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Metadata" + }, + "naming": { + "anyOf": [ + { + "$ref": "#/$defs/BlueprintNaming" + }, + { + "type": "null" + } + ], + "default": null + }, + "snmp": { + "anyOf": [ + { + "$ref": "#/$defs/SnmpSelection" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "InventoryConfig", + "type": "object" + }, + "Join": { + "additionalProperties": false, + "description": "Join rendered parts with ``separator``. With ``skip_missing``, parts whose *optional* value is\nabsent are omitted; unknown fields, type errors, and failed mappings are still errors.", + "properties": { + "join": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Text" + }, + { + "$ref": "#/$defs/Field" + }, + { + "$ref": "#/$defs/Join" + } + ] + }, + "title": "Join", + "type": "array" + }, + "separator": { + "default": "", + "title": "Separator", + "type": "string" + }, + "skip_missing": { + "default": false, + "title": "Skip Missing", + "type": "boolean" + } + }, + "required": [ + "join" + ], + "title": "Join", + "type": "object" + }, + "ModulePatch": { + "additionalProperties": false, + "description": "Module-local tag settings (module targets only).", + "properties": { + "tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "title": "ModulePatch", + "type": "object" + }, + "SnmpSelection": { + "additionalProperties": false, + "description": "Global SNMP selection. ``configuration`` is an exact label (string) or ``{id: ...}``.", + "properties": { + "configuration": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/CatalogId" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Configuration" + }, + "use_global_settings": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use Global Settings" + } + }, + "title": "SnmpSelection", + "type": "object" + }, + "TagDelta": { + "additionalProperties": false, + "description": "Additive tag change that preserves unrelated local tags. Tag references are exact catalog\ntag ids (for example ``Format~~V_1080p50``); the engine never creates catalog entries.", + "properties": { + "add": { + "items": { + "minLength": 1, + "type": "string" + }, + "title": "Add", + "type": "array" + }, + "remove": { + "items": { + "minLength": 1, + "type": "string" + }, + "title": "Remove", + "type": "array" + } + }, + "title": "TagDelta", + "type": "object" + }, + "Text": { + "additionalProperties": false, + "description": "A literal text block: ``Text(\"-\")`` or YAML ``{text: \"-\"}``.", + "properties": { + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Text", + "type": "object" + }, + "TopologyConfig": { + "additionalProperties": false, + "description": "One topology entry (``default`` or a named variant patch).", + "properties": { + "device": { + "anyOf": [ + { + "$ref": "#/$defs/DevicePatch" + }, + { + "type": "null" + } + ], + "default": null + }, + "ip_vertex_mapping": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "type": "array" + }, + "propertyNames": { + "minLength": 1 + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ip Vertex Mapping" + }, + "module": { + "anyOf": [ + { + "$ref": "#/$defs/ModulePatch" + }, + { + "type": "null" + } + ], + "default": null + }, + "naming": { + "anyOf": [ + { + "$ref": "#/$defs/BlueprintNaming" + }, + { + "type": "null" + } + ], + "default": null + }, + "vertex_processor": { + "anyOf": [ + { + "$ref": "#/$defs/VertexProcessorSpec" + }, + { + "type": "null" + } + ], + "default": null + }, + "vertices": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/VertexOverride" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vertices" + } + }, + "title": "TopologyConfig", + "type": "object" + }, + "VertexOverride": { + "additionalProperties": false, + "description": "Explicit vertex edit. Select by exact ``vertex_id``, or by exact ``factory_label`` (supports\n``{module.position}``) optionally restricted by ``kind`` / ``direction``; must match exactly one vertex.", + "properties": { + "direction": { + "anyOf": [ + { + "enum": [ + "In", + "Out", + "Internal", + "Undecided" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Direction" + }, + "factory_label": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Factory Label" + }, + "fields": { + "$ref": "#/$defs/VertexPatch" + }, + "kind": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Kind" + }, + "vertex_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vertex Id" + } + }, + "required": [ + "fields" + ], + "title": "VertexOverride", + "type": "object" + }, + "VertexPatch": { + "additionalProperties": false, + "description": "Supported vertex edits. ``\"\"`` is an explicit clear for label/description.\n\nKind-specific fields: ``sdp_support`` and the destination ports apply to codec vertices,\n``supports_static_igmp`` to IP vertices.", + "properties": { + "active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Active" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Label" + }, + "main_destination_port": { + "anyOf": [ + { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Main Destination Port" + }, + "sdp_support": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sdp Support" + }, + "sips_mode": { + "anyOf": [ + { + "enum": [ + "NONE", + "SIPSAuto", + "SIPSDuplicate", + "SIPSMerge", + "SIPSSplit" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sips Mode" + }, + "spare_destination_port": { + "anyOf": [ + { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spare Destination Port" + }, + "supports_static_igmp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Supports Static Igmp" + }, + "tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "use_as_endpoint": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use As Endpoint" + } + }, + "title": "VertexPatch", + "type": "object" + }, + "VertexProcessorSpec": { + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "processor_type": { + "const": "matrox.convertip.default" + } + }, + "required": [ + "processor_type" + ] + }, + "then": { + "properties": { + "params": { + "additionalProperties": false, + "description": "Parameters. Omitted tag parameters leave local tags unchanged (``[]`` clears them); an omitted\n``redundant_streams`` preserves the SIPS mode (``true`` \u2192 ``SIPSAuto``, ``false`` \u2192 ``NONE``).", + "properties": { + "allow_null_audio_label": { + "default": false, + "title": "Allow Null Audio Label", + "type": "boolean" + }, + "audio_receiver_tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Receiver Tags" + }, + "audio_sender_tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Sender Tags" + }, + "configure_control": { + "default": false, + "title": "Configure Control", + "type": "boolean" + }, + "mode": { + "default": "auto", + "enum": [ + "auto", + "tx", + "rx", + "four_split" + ], + "title": "Mode", + "type": "string" + }, + "redundant_streams": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Redundant Streams" + }, + "video_receiver_tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Video Receiver Tags" + }, + "video_sender_tags": { + "anyOf": [ + { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Video Sender Tags" + } + }, + "title": "MatroxConvertIPParams", + "type": "object" + } + } + } + } + ], + "description": "Processor declaration. ``processor_type`` may be inherited from ``default`` by a variant;\nit is required on the resolved configuration.", + "properties": { + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + }, + "processor_type": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Processor Type" + } + }, + "title": "VertexProcessorSpec", + "type": "object" + } + }, + "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "A reusable, type-level configuration with ``default`` entries and optional named variants.\n\nLoad with :meth:`load` (file), :meth:`from_yaml` (text/bytes), or :meth:`from_dict` (mapping).\nLoading never connects to a server. A named variant is a patch over its section's ``default``.", + "properties": { + "inventory": { + "anyOf": [ + { + "patternProperties": { + "^[A-Za-z0-9][A-Za-z0-9_.-]*$": { + "$ref": "#/$defs/InventoryConfig" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory" + }, + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "topology": { + "anyOf": [ + { + "patternProperties": { + "^[A-Za-z0-9][A-Za-z0-9_.-]*$": { + "$ref": "#/$defs/TopologyConfig" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Topology" + } + }, + "required": [ + "schema_version" + ], + "title": "VideoIPath Automation Tool blueprint (schema_version 1)", + "type": "object" +} diff --git a/tests/blueprints/__init__.py b/tests/blueprints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/blueprints/conftest.py b/tests/blueprints/conftest.py new file mode 100644 index 0000000..e1ea743 --- /dev/null +++ b/tests/blueprints/conftest.py @@ -0,0 +1,365 @@ +"""Offline harness for blueprint tests. + +- ``FakeInventory`` implements the handful of ``InventoryApp`` methods the engine uses, backed by + in-memory :class:`InventoryDevice` records. +- ``FakeInspectServer`` is an in-memory stand-in for ``InspectAPI``; it sits behind a *real* + ``InspectApp`` so transactions, conflict checks, and snapshot refreshes run unchanged. +- ``matrox_layout`` builds synthetic (anonymized) ConvertIP topologies: ``tx``, ``rx``, ``four_split``. +""" + +from __future__ import annotations + +import copy +import logging +from types import SimpleNamespace +from typing import Any + +import pytest + +from videoipath_automation_tool.apps.inspect.app import InspectApp +from videoipath_automation_tool.apps.inspect.model.actions import ( + InspectApiLookupInspectDeviceResponse, + InspectApiLookupSyncInfoResponse, + InspectApiLookupVerticesResponse, +) +from videoipath_automation_tool.apps.inspect.model.collector import InspectApiNodeStatusItem +from videoipath_automation_tool.apps.inspect.model.common import InspectApiSimpleActionResponse +from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyResponse +from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice +from videoipath_automation_tool.blueprints import BlueprintEngine + +NMOS = "com.nevion.NMOS_multidevice-0.1.0" +HEADER = {"auth": True, "caption": "OK", "code": "OK", "id": "0", "ok": True, "user": "test-user"} + + +# --- Inventory --- + + +class FakeInventory: + def __init__(self) -> None: + self.devices: dict[str, InventoryDevice] = {} + self.snmp: dict[str, str] = {"default": "Default configuration", "snmp-1": "snmp-a"} + self.writes: list[tuple[str, str]] = [] + self.address_lookups = 0 + self.fail_next_write: Exception | None = None + self._next_id = 100 + + def seed(self, device_id: str, **fields: Any) -> InventoryDevice: + device = InventoryDevice.create(fields.pop("driver_id", NMOS)) + device.configuration.id = device_id + device.configuration.config.desc.label = fields.pop("label", device_id) + device.configuration.config.cinfo.address = fields.pop("address", "192.0.2.1") + for name, value in fields.pop("custom", {}).items(): + setattr(device.configuration.config.customSettings, name, value) + assert not fields, fields + self.devices[device_id] = device + return device + + def get_device(self, device_id: str, config_only: bool = False) -> InventoryDevice: + if device_id not in self.devices: + raise ValueError(f"No device with id '{device_id}' found in Inventory.") + return self.devices[device_id].model_copy(deep=True) + + def add_device(self, device: InventoryDevice, label_check: bool, address_check: bool, config_only: bool) -> Any: + assert not label_check and not address_check and config_only + self._maybe_fail() + device_id = f"device{self._next_id}" + self._next_id += 1 + stored = device.model_copy(deep=True) + stored.configuration.id = device_id + self.devices[device_id] = stored + self.writes.append(("add", device_id)) + return stored.model_copy(deep=True) + + def update_device(self, device: InventoryDevice, compare_config: bool, config_only: bool) -> Any: + assert compare_config is False and config_only + self._maybe_fail() + self.devices[device.device_id] = device.model_copy(deep=True) + self.writes.append(("update", device.device_id)) + return device.model_copy(deep=True) + + def find_device_id_by_label(self, label: str, label_search_mode: str) -> Any: + ids = [i for i, d in self.devices.items() if d.configuration.config.desc.label == label] + return None if not ids else ids[0] if len(ids) == 1 else ids + + def get_global_snmp_config_id_by_label(self, label: str) -> Any: + ids = [i for i, lbl in self.snmp.items() if lbl == label] + return None if not ids else ids[0] if len(ids) == 1 else ids + + def get_global_snmp_config_label_by_id(self, snmp_config_id: str) -> str | None: + return self.snmp.get(snmp_config_id) + + def find_device_ids_by_addresses(self, addresses: Any) -> list[str]: + self.address_lookups += 1 + ids = [ + i + for i, device in self.devices.items() + if any( + address == device.configuration.config.cinfo.address + or address in device.configuration.config.cinfo.altAddresses + for address in addresses + ) + ] + return sorted(set(ids)) + + def _maybe_fail(self) -> None: + if self.fail_next_write is not None: + error, self.fail_next_write = self.fail_next_write, None + raise error + + +# --- Inspect --- + + +class FakeInspectServer: + """In-memory Inspect API (only the endpoints the engine and transactions use).""" + + def __init__(self) -> None: + self.nodes: dict[str, dict[str, Any]] = {} + self.device_forms: dict[str, dict[str, Any]] = {} + self.vertex_forms: dict[str, dict[str, Any]] = {} + self.sync: dict[str, dict[str, Any]] = {} + self.discoverable: dict[str, dict[str, Any]] = {} + self.calls: list[str] = [] + self.writes: list[tuple[str, Any]] = [] + self.fail_commit: Exception | None = None + self.fail_tag: str | None = None + self.fail_sync_info: Exception | None = None + + # Setup + + def install(self, layout: dict[str, Any]) -> None: + self.nodes[layout["device_id"]] = copy.deepcopy(layout["node"]) + self.device_forms[layout["device_id"]] = copy.deepcopy(layout["device_form"]) + self.vertex_forms.update(copy.deepcopy(layout["vertex_forms"])) + + # Reads + + def get_device_detail(self, device_id: str) -> InspectApiNodeStatusItem | None: + self.calls.append(f"detail:{device_id}") + node = self.nodes.get(device_id) + return InspectApiNodeStatusItem.model_validate(copy.deepcopy(node)) if node is not None else None + + def lookup_inspect_device(self, device_id: str) -> InspectApiLookupInspectDeviceResponse: + self.calls.append(f"lookup_device:{device_id}") + if device_id not in self.device_forms: + raise ValueError(f"unknown device {device_id}") + data = {"assignedTags": {}, "fields": copy.deepcopy(self.device_forms[device_id])} + return InspectApiLookupInspectDeviceResponse.model_validate({"data": data, "header": HEADER}) + + def lookup_vertices(self, vertex_ids: list[str]) -> InspectApiLookupVerticesResponse: + self.calls.append(f"lookup_vertices:{len(vertex_ids)}") + data = {vid: copy.deepcopy(self.vertex_forms[vid]) for vid in vertex_ids if vid in self.vertex_forms} + return InspectApiLookupVerticesResponse.model_validate({"data": data, "header": HEADER}) + + def lookup_sync_info(self, device_ids: list[str]) -> InspectApiLookupSyncInfoResponse: + self.calls.append("sync_info") + if self.fail_sync_info is not None: + error, self.fail_sync_info = self.fail_sync_info, None + raise error + data = {d: {"label": d, **self.sync[d]} for d in device_ids if d in self.sync} + return InspectApiLookupSyncInfoResponse.model_validate({"data": data, "header": HEADER}) + + def get_device_skeleton(self) -> list[InspectApiNodeStatusItem]: + return [InspectApiNodeStatusItem.model_validate({"_id": d}) for d in self.nodes] + + def get_edge_skeleton(self) -> list[Any]: + return [] + + # Writes + + def update_topology(self, delta: Any) -> InspectApiUpdateTopologyResponse: + self.writes.append(("update_topology", delta.model_dump(mode="json"))) + if self.fail_commit is not None: + error, self.fail_commit = self.fail_commit, None + raise error + for device_id, form in delta.replaceDevices.items(): + self.device_forms[device_id] = form.model_dump(mode="json") + for vertex_id, form in delta.replaceVertices.items(): + self.vertex_forms[vertex_id]["fields"] = form.model_dump(mode="json") + result = {"msg": [], "ok": True} + data = {"items": [], "res": result, "validation": {"createIds": [], "details": {}, "result": result}} + return InspectApiUpdateTopologyResponse.model_validate({"data": data, "header": HEADER}) + + def assign_tag(self, tag_id: str, element_ids: list[str]) -> InspectApiSimpleActionResponse: + return self._tag("assign", tag_id, element_ids) + + def unassign_tag(self, tag_id: str, element_ids: list[str]) -> InspectApiSimpleActionResponse: + return self._tag("unassign", tag_id, element_ids) + + def add_devices(self, items: list[Any]) -> InspectApiSimpleActionResponse: + for item in items: + self.writes.append(("add_devices", item.id)) + pending = self.discoverable.get(item.id) + if pending is None: + return _action(False, ["device not discovered"]) + self.nodes[item.id] = {"_id": item.id, "modules": {}} + self.device_forms[item.id] = copy.deepcopy(pending["device_form"]) + self.sync[item.id] = {"add": {"modules": 1}, "update": {}, "remove": {}} + return _action(True) + + def sync_devices(self, device_ids: list[str], add_only: bool = True, conflict_strategy: int = 0) -> Any: + for device_id in device_ids: + self.writes.append(("sync_devices", (device_id, add_only))) + pending = self.discoverable.pop(device_id, None) + if pending is not None: + self.install(pending) + self.sync.pop(device_id, None) + return _action(True) + + def _tag(self, action: str, tag_id: str, element_ids: list[str]) -> InspectApiSimpleActionResponse: + self.writes.append((action, (tag_id, tuple(element_ids)))) + if self.fail_tag == tag_id: + return _action(False, ["tag rejected"]) + for element in element_ids: + module_id = element.removeprefix("device:") + device_id = module_id.split(".", 1)[0] + module = self.nodes[device_id]["modules"][module_id] + local = module.setdefault("tagsInfo", {}).setdefault("assigned", {}).setdefault("local", {}) + if action == "assign": + local[tag_id] = {} + else: + local.pop(tag_id, None) + return _action(True) + + +def _action(ok: bool, msg: list[str] | None = None) -> InspectApiSimpleActionResponse: + return InspectApiSimpleActionResponse.model_validate({"data": {"ok": ok, "msg": msg or []}, "header": HEADER}) + + +def make_inspect_app(server: FakeInspectServer) -> InspectApp: + app = InspectApp.__new__(InspectApp) + app._logger = logging.getLogger("test-inspect") + app._inspect_api = server + app._vip_connector = None + app._load_mode = "skeleton" + app._snapshot = None + return app + + +# --- Synthetic layouts --- + + +def vertex_form(vertex_id: str, vertex_type: str, kind: str, **fields: Any) -> dict[str, Any]: + type_fields: dict[str, Any] = {"type": kind} + if kind == "codec": + media = fields.pop("media") + type_fields["generic"] = {"codecFormat": media.capitalize() if media else None, "multiplicity": 1} + type_fields["specific"] = {"type": media, "sdpSupport": False, "isIgmpSource": False} + form = { + "active": True, + "controlProps": {"configPriority": "off", "onlyInitial": False}, + "custom": {}, + "desc": "", + "label": fields.pop("label", ""), + "localAssignedTags": fields.pop("tags", []), + "sipsMode": "NONE", + "tags": [], + "typeFields": type_fields, + "useAsEndpoint": fields.pop("use_as_endpoint", False), + } + assert not fields, fields + return {"id": vertex_id, "vertexType": vertex_type, "fields": form} + + +def build_layout( + device_id: str, modules: dict[str, list[tuple[str, str, list[tuple[str, str, str, str | None]]]]] +) -> dict[str, Any]: + """``modules``: module id -> [(port suffix, factory label, [(vertex suffix, type, kind, media)])].""" + node_modules: dict[str, Any] = {} + forms: dict[str, Any] = {} + for module_id, ports in modules.items(): + node_ports: dict[str, Any] = {} + for port_suffix, factory_label, vertices in ports: + port_id = f"{module_id}.{port_suffix}" + sides = [] + for vertex_suffix, vertex_type, kind, media in vertices: + vertex_id = f"{device_id}.{module_id.split('.')[-1]}.{port_suffix}.{vertex_suffix}" + sides.append( + { + "type": "single", + "id": vertex_id, + "label": factory_label, + "vertexType": vertex_type, + "fields": {"isActive": True, "isControlled": False, "isEndpoint": False}, + } + ) + forms[vertex_id] = vertex_form( + vertex_id, vertex_type, kind, **({"media": media} if kind == "codec" else {}) + ) + if len(sides) == 1: + info: dict[str, Any] = sides[0] + else: + info = {"type": "double", **{("in" if s["vertexType"] == "In" else "out"): s for s in sides}} + node_ports[port_id] = {"pid": port_id, "label": factory_label, "vertexInfo": info} + node_modules[module_id] = { + "pid": module_id, + "label": f"Slot {module_id}", + "ports": node_ports, + "tagsInfo": {"assigned": {"all": [], "local": {}}}, + } + return { + "device_id": device_id, + "node": {"_id": device_id, "fDescriptor": {"label": "factory-device", "desc": ""}, "modules": node_modules}, + "device_form": { + "coordinates": {"x": 0, "y": 0}, + "descriptor": {"label": "", "desc": ""}, + "iconSize": "auto", + "iconType": "default", + "localAssignedTags": [], + "sdpStrategy": "always", + "siteId": None, + "tags": [], + }, + "vertex_forms": forms, + } + + +def matrox_layout(device_id: str = "device1", mode: str = "tx", module: str | None = None) -> dict[str, Any]: + module_id = module or f"{device_id}.dev.0" + ip_ports = [ + ("P1", "P1", [("in", "In", "ip", None), ("out", "Out", "ip", None)]), + ("P2", "P2", [("in", "In", "ip", None), ("out", "Out", "ip", None)]), + ("MGMT", "MGMT", [("in", "In", "ip", None), ("out", "Out", "ip", None)]), + ] + if mode == "tx": + codecs = [ + ("vs", "Video Sender", [("v", "In", "codec", "video")]), + ("as", "Audio Sender", [("v", "In", "codec", "audio")]), + ] + elif mode == "rx": + codecs = [ + ("vr", "Video Receiver", [("v", "Out", "codec", "video")]), + ("ar", "Audio Receiver", [("v", "Out", "codec", "audio")]), + ] + else: + codecs = [(f"vr{i}", f"Video Receiver {i}", [("v", "Out", "codec", "video")]) for i in range(4)] + codecs.append(("ar", "Audio Receiver", [("v", "Out", "codec", "audio")])) + return build_layout(device_id, {module_id: codecs + ip_ports}) + + +# --- Fixtures --- + + +@pytest.fixture +def inventory() -> FakeInventory: + return FakeInventory() + + +@pytest.fixture +def server() -> FakeInspectServer: + return FakeInspectServer() + + +@pytest.fixture +def app(inventory: FakeInventory, server: FakeInspectServer) -> SimpleNamespace: + return SimpleNamespace(inventory=inventory, inspect=make_inspect_app(server)) + + +@pytest.fixture +def engine(app: SimpleNamespace) -> BlueprintEngine: + engine = BlueprintEngine(app) + clock = {"now": 0.0} + engine._clock = lambda: clock["now"] + engine._sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) + return engine diff --git a/tests/blueprints/test_blueprint_inputs.py b/tests/blueprints/test_blueprint_inputs.py new file mode 100644 index 0000000..aba1d15 --- /dev/null +++ b/tests/blueprints/test_blueprint_inputs.py @@ -0,0 +1,106 @@ +"""Blueprint entry points load file inputs and retain reviewed content for later application.""" + +from __future__ import annotations + +from pathlib import Path, PurePath +from types import SimpleNamespace + +import pytest + +from tests.blueprints.conftest import NMOS, FakeInventory +from videoipath_automation_tool.blueprints import Blueprint, BlueprintDevice, BlueprintEngine, BlueprintValidationError + + +@pytest.fixture +def blueprint_path(tmp_path: Path) -> Path: + path = tmp_path / "device-blueprint.yml" + path.write_text( + f"schema_version: 1\ninventory:\n default:\n driver_id: {NMOS}\n custom_settings: {{port: 8080}}\n", + encoding="utf-8", + ) + return path + + +@pytest.mark.parametrize("source_type", [str, Path, PurePath], ids=["string", "path", "pathlike"]) +def test_file_inputs_match_blueprint_instances( + blueprint_path: Path, inventory: FakeInventory, source_type: type[str | PurePath] +) -> None: + inventory.seed("device1", label="device-a", custom={"port": 80}) + engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) + device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + source = source_type(blueprint_path) + + engine.validate(source) + expected = engine.plan(device, Blueprint.load(blueprint_path), scope="inventory") + actual = engine.plan(device, source, scope="inventory") + assert actual.model_dump() == expected.model_dump() + assert inventory.writes == [] + + result = engine.apply(device, source, scope="inventory") + assert result.status == "succeeded" + assert inventory.devices["device1"].configuration.custom_settings.port == 8080 + + +def test_apply_loads_file_once(blueprint_path: Path, inventory: FakeInventory, monkeypatch: pytest.MonkeyPatch) -> None: + inventory.seed("device1", label="device-a", custom={"port": 80}) + engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) + device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + original_load = Blueprint.load + loaded: list[Path] = [] + + def load_once(path: Path) -> Blueprint: + loaded.append(path) + return original_load(path) + + monkeypatch.setattr(Blueprint, "load", load_once) + engine.apply(device, blueprint_path, scope="inventory") + assert loaded == [blueprint_path] + + +def test_plans_keep_loaded_content_when_file_changes_or_disappears( + blueprint_path: Path, inventory: FakeInventory +) -> None: + inventory.seed("device1", label="device-a", custom={"port": 80}) + engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) + device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + + first = engine.plan(device, blueprint_path, scope="inventory") + blueprint_path.write_text(blueprint_path.read_text(encoding="utf-8").replace("8080", "9090"), encoding="utf-8") + first.apply() + assert inventory.devices["device1"].configuration.custom_settings.port == 8080 + + second = engine.plan(device, blueprint_path, scope="inventory") + assert second.blueprint_digest != first.blueprint_digest + blueprint_path.unlink() + second.apply() + assert inventory.devices["device1"].configuration.custom_settings.port == 9090 + + +@pytest.mark.parametrize("method", ["plan", "apply", "validate"]) +def test_file_errors_happen_before_app_access( + blueprint_path: Path, method: str, monkeypatch: pytest.MonkeyPatch +) -> None: + engine = BlueprintEngine(SimpleNamespace()) + device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + operation = getattr(engine, method) + args = () if method == "validate" else (device,) + monkeypatch.chdir(blueprint_path.parent) + + with pytest.raises(FileNotFoundError): + operation(*args, "missing.yml") + with pytest.raises(FileNotFoundError): + operation(*args, "schema_version: 1") # Strings are paths, not implicit YAML documents. + + blueprint_path.write_text("schema_version: 1\nschema_version: 1\n", encoding="utf-8") + with pytest.raises(BlueprintValidationError, match="duplicate key") as error: + operation(*args, blueprint_path.name) + assert blueprint_path.name in str(error.value) + + +@pytest.mark.parametrize("method", ["plan", "apply", "validate"]) +def test_invalid_blueprint_input_has_a_useful_type_error(method: str) -> None: + engine = BlueprintEngine(SimpleNamespace()) + device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + args = () if method == "validate" else (device,) + with pytest.raises(TypeError, match="Blueprint instance or a YAML file path"): + getattr(engine, method)(*args, {"schema_version": 1}) diff --git a/tests/blueprints/test_engine.py b/tests/blueprints/test_engine.py new file mode 100644 index 0000000..daf4e60 --- /dev/null +++ b/tests/blueprints/test_engine.py @@ -0,0 +1,947 @@ +"""Engine configuration, read-only planning, execution phases, conflicts, and partial failures.""" + +from __future__ import annotations + +import inspect +import subprocess +import sys +from types import SimpleNamespace +from typing import Any + +import pytest +from pydantic import BaseModel, ConfigDict + +from tests.blueprints.conftest import NMOS, FakeInspectServer, FakeInventory, make_inspect_app, matrox_layout +from videoipath_automation_tool.apps.inspect.snapshot import InspectSnapshot +from videoipath_automation_tool.blueprints import ( + AlternativeAddress, + ApplyOptions, + Blueprint, + BlueprintApplyError, + BlueprintCapabilityError, + BlueprintConflictError, + BlueprintDevice, + BlueprintEngine, + BlueprintError, + BlueprintPlan, + BlueprintTargetError, + BlueprintValidationError, + Credentials, + DeviceTarget, + EndpointIdentity, + ModuleTarget, + NamingScheme, + ProcessingContext, + ProcessorInputError, + ProcessorResult, + VertexEdit, + VertexPatch, + VertexProcessor, + published_json_schema, +) + +MATROX = { + "schema_version": 1, + "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": 8080}}}, + "topology": { + "default": { + "device": {"icon_size": "medium"}, + "ip_vertex_mapping": {"stream-a": ["P1"], "stream-b": ["P2"]}, + "vertex_processor": { + "processor_type": "matrox.convertip.default", + "params": {"redundant_streams": True, "video_sender_tags": ["video-tag-b"]}, + }, + }, + "receiver": {"vertex_processor": {"params": {"mode": "rx"}}}, + }, +} + + +def _blueprint(document: dict[str, Any] = MATROX, **topology_default: Any) -> Blueprint: + data = { + **document, + "topology": {**document["topology"], "default": {**document["topology"]["default"], **topology_default}}, + } + return Blueprint.from_dict(data) + + +def _device(**fields: Any) -> BlueprintDevice: + return BlueprintDevice(**{"key": "key-a", "label": "device-a", "inventory_id": "device1", **fields}) + + +def _existing(inventory: FakeInventory, server: FakeInspectServer, mode: str = "tx") -> None: + inventory.seed("device1", label="device-a", custom={"port": 8080, "indices_in_ids": False}) + server.install(matrox_layout("device1", mode)) + + +# --- Construction and configuration --- + + +class _ExplodingApp: + @property + def inventory(self) -> Any: + raise AssertionError("inventory accessed") + + @property + def inspect(self) -> Any: + raise AssertionError("inspect accessed") + + +class _Params(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + factory_label: str + + +class SimpleVideoProcessor(VertexProcessor[_Params]): + params_model = _Params + + def process(self, context: ProcessingContext, params: _Params) -> ProcessorResult: + vertex = context.require_vertex(kind="codec", factory_label=params.factory_label, vertex_type="In") + return ProcessorResult( + vertices=[ + VertexEdit( + vertex_id=vertex.id, + fields=VertexPatch(use_as_endpoint=True), + endpoint=EndpointIdentity(direction="TX", media="video", index=1), + ) + ] + ) + + +def test_construction_is_lazy_and_injection_matches_configuration() -> None: + options = ApplyOptions(sync="none", discovery_timeout=60) + naming = NamingScheme(endpoint_label="{device.label}-{endpoint.index:02d}") + injected = BlueprintEngine( + _ExplodingApp(), processors={"example-org.simple": SimpleVideoProcessor}, options=options, naming=naming + ) + configured = BlueprintEngine(_ExplodingApp()) + configured.register_processor("example-org.simple", SimpleVideoProcessor) + configured.configure(options=options) + configured.configure(naming=naming) + assert (injected.options, injected.naming, injected.registry.ids()) == ( + configured.options, + configured.naming, + configured.registry.ids(), + ) + configured.configure(options=None) + assert configured.options == ApplyOptions() and configured.naming == naming + with pytest.raises(ValueError, match="already registered"): + configured.register_processor("example-org.simple", SimpleVideoProcessor) + with pytest.raises(TypeError): + configured.configure(options={"sync": "none"}) # type: ignore[arg-type] + assert "example-org.simple" not in BlueprintEngine(_ExplodingApp()).registry + + +def test_apply_has_the_plan_signature_plus_dry_run() -> None: + plan_parameters = list(inspect.signature(BlueprintEngine.plan).parameters.values()) + apply_parameters = list(inspect.signature(BlueprintEngine.apply).parameters.values()) + assert apply_parameters[:-1] == plan_parameters + assert (apply_parameters[-1].name, apply_parameters[-1].default) == ("dry_run", False) + assert inspect.signature(BlueprintPlan.apply).parameters["dry_run"].default is False + + +def test_normal_app_import_does_not_load_blueprints() -> None: + code = "import sys, videoipath_automation_tool; print('videoipath_automation_tool.blueprints' in sys.modules, 'yaml' in sys.modules)" + output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout.split() + assert output == ["False", "False"] + + +# --- Planning --- + + +def test_planning_is_read_only( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + plan = engine.plan(_device(), _blueprint()) + assert plan.fully_resolved and plan.has_changes + assert inventory.writes == [] and server.writes == [] + assert app.inspect._snapshot is None + assert [phase.status for phase in plan.phases] == ["planned", "skipped", "no_change", "planned", "skipped"] + assert plan.phase("inventory").operations[0].changes[0].model_dump() == { + "field": "label", + "before": "old", + "after": "device-a", + "source": "naming", + "sensitive": False, + } + assert [b.port_id for b in plan.interface_bindings] == ["device1.dev.0.P1", "device1.dev.0.P2"] + + +def test_inventory_only_scope_never_touches_inspect(inventory: FakeInventory) -> None: + inventory.seed("device1", label="device-a") + app = SimpleNamespace(inventory=inventory) + document = {**MATROX, "topology": {"default": {"vertex_processor": {"processor_type": "example-org.missing"}}}} + result = BlueprintEngine(app).apply(_device(), Blueprint.from_dict(document), scope="inventory") + assert result.status == "succeeded" and result.phase("topology").status == "skipped" + assert inventory.devices["device1"].configuration.config.customSettings.port == 8080 + + +def test_apply_then_reapply_converges( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + result = engine.apply(_device(), _blueprint()) + assert result.status == "succeeded" and result.verification == "confirmed" + assert [name for name, _ in server.writes] == ["update_topology"] + labels = {f["fields"]["label"] for f in server.vertex_forms.values() if f["fields"]["useAsEndpoint"]} + assert labels == {"device-a-TX-video-01", "device-a-TX-audio-01"} + + server.writes.clear() + again = engine.apply(_device(), _blueprint()) + assert again.status == "no_change" and again.verification == "not_applicable" + assert server.writes == [] and inventory.writes == [] + + +def test_both_usage_styles_execute_the_same_workflow() -> None: + outcomes = [] + for style in ("direct", "two-step"): + inventory, server = FakeInventory(), FakeInspectServer() + _existing(inventory, server) + engine = BlueprintEngine(SimpleNamespace(inventory=inventory, inspect=make_inspect_app(server))) + if style == "direct": + result = engine.apply(_device(), _blueprint(), topology_variant="default") + else: + result = engine.plan(_device(), _blueprint(), topology_variant="default").apply() + outcomes.append((result.model_dump(), server.writes, inventory.writes)) + assert outcomes[0] == outcomes[1] + + +def test_update_preserves_unmanaged_fields_and_redacts_secrets( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + document = {**MATROX, "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}}} + device = _device( + credentials=Credentials(username="test-user", password="test-password"), + alternative_addresses=[ + "192.0.2.20", + AlternativeAddress( + address="192.0.2.21", credentials=Credentials(username="test-user", password="alt-password") + ), + ], + ) + plan = engine.plan(device, Blueprint.from_dict(document), scope="inventory") + summary = plan.summary() + assert "test-password" not in summary and "alt-password" not in summary and "********" in summary + assert "test-password" not in repr(device) and "test-password" not in plan.model_dump_json() + plan.apply() + stored = inventory.devices["device1"].configuration + assert stored.config.customSettings.port == 8080 and stored.config.customSettings.indices_in_ids is False + assert stored.config.customSettings.disable_rx_sdp is True + assert (stored.config.cinfo.auth.user, stored.config.cinfo.auth.password) == ("test-user", "test-password") + assert stored.config.cinfo.altAddresses == ["192.0.2.20", "192.0.2.21"] + assert stored.config.cinfo.altAddressesWithAuth == [ + {"address": "192.0.2.21", "authentication": {"user": "test-user", "password": "alt-password"}} + ] + # A changed password alone still writes; omission preserves it. + changed = device.model_copy(update={"credentials": Credentials(username="test-user", password="new-password")}) + assert engine.plan(changed, Blueprint.from_dict(document), scope="inventory").phase("inventory").status == "planned" + assert ( + engine.plan(_device(), Blueprint.from_dict(document), scope="inventory").phase("inventory").status + == "no_change" + ) + + +def test_api_key_is_redacted_in_the_plan(engine: BlueprintEngine) -> None: + secret = "synthetic-key-value" + document = { + "schema_version": 1, + "inventory": { + "default": { + "driver_id": "com.nevion.spg9000-0.1.0", + "custom_settings": {"x_api_key": secret}, + } + }, + } + device = BlueprintDevice(key="key-a", label="device-a", management_address="192.0.2.40") + plan = engine.plan(device, Blueprint.from_dict(document), scope="inventory") + change = next( + item for item in plan.phase("inventory").operations[0].changes if item.field == "custom_settings.x_api_key" + ) + assert change.sensitive and change.after == "********" + assert secret not in plan.summary() + assert secret not in plan.model_dump_json() + + +def test_inventory_binding_rules(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + inventory.seed("device1", label="device-a", address="192.0.2.10") + with pytest.raises(BlueprintTargetError, match="'device9' was not found"): + engine.plan(_device(inventory_id="device9"), _blueprint(), scope="inventory") + new = BlueprintDevice(key="key-b", label="device-a", management_address="192.0.2.10") + with pytest.raises(BlueprintTargetError, match="used by device1.*Bind the existing record"): + engine.plan(new, _blueprint(), scope="inventory") + assert inventory.address_lookups == 1 + with pytest.raises(BlueprintTargetError, match="requires 'management_address'"): + engine.plan(BlueprintDevice(key="key-b", label="device-b"), _blueprint(), scope="inventory") + other_driver = {**MATROX, "inventory": {"default": {"driver_id": "com.nevion.NMOS-0.1.0"}}} + with pytest.raises(BlueprintCapabilityError, match="Driver migration"): + engine.plan(_device(), Blueprint.from_dict(other_driver), scope="inventory") + + +def test_snmp_references_resolve_exactly(engine: BlueprintEngine, inventory: FakeInventory) -> None: + inventory.seed("device1", label="device-a") + + def plan_with(reference: Any) -> Any: + document = {**MATROX, "inventory": {"default": {"driver_id": NMOS, "snmp": {"configuration": reference}}}} + return engine.plan(_device(), Blueprint.from_dict(document), scope="inventory") + + assert plan_with("snmp-a").phase("inventory").operations[0].changes[0].after == "snmp-1" + assert plan_with({"id": "default"}).phase("inventory").status == "no_change" + with pytest.raises(BlueprintTargetError, match="No SNMP configuration"): + plan_with("missing") + inventory.snmp["snmp-2"] = "snmp-a" + with pytest.raises(BlueprintTargetError, match="ambiguous"): + plan_with("snmp-a") + + +# --- Creation, discovery, and synchronization --- + + +def test_create_discover_and_configure( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + plan = engine.plan(device, _blueprint()) + assert not plan.fully_resolved and plan.phase("topology").status == "deferred" + assert "materialized during apply()" in plan.summary() + + server.discoverable["device100"] = matrox_layout("device100", "tx") + result = plan.apply() + assert result.status == "succeeded" and result.materialized + assert (result.inventory_id, result.topology_device_id) == ("device100", "device100") + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices", "update_topology"] + assert [op.action for op in result.phase("topology_sync").operations] == ["add_to_topology", "sync"] + assert server.vertex_forms["device100.0.vs.v"]["fields"]["label"] == "device-new-TX-video-01" + + +def test_discovery_timeout_reports_created_id(engine: BlueprintEngine, server: FakeInspectServer) -> None: + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(BlueprintApplyError) as info: + engine.apply(device, _blueprint(), options=ApplyOptions(discovery_timeout=3, poll_interval=1)) + result = info.value.result + assert result.status == "partial" and result.inventory_id == "device100" + assert result.phase("inventory").status == "completed" + assert result.phase("discovery").status == "failed" + assert "not ready after 3s" in result.phase("discovery").message + assert [name for name, _ in server.writes].count("add_devices") == 4 + + +def test_sync_policies(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + inventory.seed("device1", label="device-a", custom={"port": 8080}) + with pytest.raises(BlueprintTargetError, match="not in the topology"): + engine.plan(_device(), _blueprint(), options=ApplyOptions(sync="none")) + + server.install(matrox_layout("device1", "tx")) + server.sync["device1"] = {"add": {}, "update": {"x": 1}, "remove": {}} + with pytest.raises(BlueprintCapabilityError, match="add_only' is insufficient"): + engine.plan(_device(), _blueprint()) + warned = engine.plan(_device(), _blueprint(), options=ApplyOptions(sync="none")) + assert warned.fully_resolved and warned.diagnostics[0].code == "topology.sync_pending" + + result = engine.apply(_device(), _blueprint(), options=ApplyOptions(sync="reconcile")) + assert ("sync_devices", ("device1", False)) in server.writes and result.materialized + + +def test_sync_lookup_failure_fails_planning( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + server.fail_sync_info = ConnectionError("sync lookup failed") + with pytest.raises(BlueprintError, match="synchronization status") as info: + engine.plan(_device(), _blueprint()) + assert type(info.value) is BlueprintError + + +# --- Conflicts and staged edits --- + + +def test_stale_plans_are_rejected(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + plan = engine.plan(_device(), _blueprint()) + inventory.devices["device1"].configuration.config.desc.label = "edited-elsewhere" + with pytest.raises(BlueprintApplyError) as info: + plan.apply() + assert info.value.result.status == "failed" and isinstance(info.value.__cause__, BlueprintConflictError) + assert inventory.writes == [] + + inventory.devices["device1"].configuration.config.desc.label = "device-a" + plan = engine.plan(_device(), _blueprint()) + server.vertex_forms["device1.0.vs.v"]["fields"]["desc"] = "edited-elsewhere" + with pytest.raises(BlueprintApplyError, match="changed since planning"): + plan.apply() + assert server.writes == [] + + +def test_staged_edits_overlap_is_rejected_and_unrelated_edits_survive( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any +) -> None: + _existing(inventory, server) + app.inspect._snapshot = InspectSnapshot(fetcher=server, device_items=server.get_device_skeleton()) + app.inspect._snapshot.stage_edit("vertex", "device1.0.vs.v", "label", "pending") + app.inspect._snapshot.stage_edit("vertex", "device1.0.P1.in", "label", "unrelated") + with pytest.raises(BlueprintApplyError, match="Pending uncommitted Inspect edits overlap"): + engine.apply(_device(), _blueprint()) + app.inspect._snapshot.clear_staged(kind="vertex", entity_id="device1.0.vs.v") + assert engine.apply(_device(), _blueprint()).status == "succeeded" + assert app.inspect._snapshot.get_staged_edits("vertex", "device1.0.P1.in") == {"label": "unrelated"} + + +# --- Failures --- + + +@pytest.mark.parametrize( + ("error", "status"), [(ConnectionError("timeout"), "unknown"), (BlueprintConflictError("x"), "partial")] +) +def test_commit_failure_outcomes( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, error: Exception, status: str +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + server.fail_commit = error + with pytest.raises(BlueprintApplyError) as info: + engine.apply(_device(), _blueprint()) + result = info.value.result + assert result.status == status and result.phase("inventory").status == "completed" + assert result.phase("topology").status == ("unknown" if status == "unknown" else "failed") + assert info.value.__cause__ is error + + +def test_inventory_write_unknown_outcome( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + inventory.fail_next_write = TimeoutError("read timed out") + with pytest.raises(BlueprintApplyError) as info: + engine.apply(_device(), _blueprint()) + assert info.value.result.status == "unknown" and info.value.result.phase("topology").status == "not_run" + + +def test_inventory_preread_failure_is_a_known_rejection( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + inventory.fail_next_write = ValueError("Failed to retrieve existing device configuration from Inventory: timed out") + with pytest.raises(BlueprintApplyError) as info: + engine.apply(_device(), _blueprint()) + result = info.value.result + assert result.status == "failed" and result.phase("inventory").status == "failed" + assert result.verification == "not_applicable" + assert inventory.writes == [] + + +# --- Modules --- + + +def _chassis(server: FakeInspectServer, inventory: FakeInventory) -> None: + inventory.seed("device1", label="chassis-a") + layout = matrox_layout("device1", "rx", module="device1.dev.1") + sibling = matrox_layout("device1", "tx", module="device1.dev.2") + layout["node"]["modules"].update(sibling["node"]["modules"]) + layout["vertex_forms"].update(sibling["vertex_forms"]) + layout["node"]["modules"]["device1.dev.1"]["tagsInfo"]["assigned"] = { + "all": ["inherited", "keep"], + "local": {"keep": {}, "old": {}}, + } + server.install(layout) + + +def test_module_target_scope_and_tags( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + document = { + "schema_version": 1, + "topology": { + "default": { + "module": {"tags": {"add": ["new"], "remove": ["old"]}}, + "vertex_processor": {"processor_type": "matrox.convertip.default"}, + } + }, + } + module = BlueprintDevice( + key="module-a", + label="module-a", + topology=ModuleTarget(device_id="device1", module_id="device1.dev.1"), + module_position="1", + ) + result = engine.apply(module, Blueprint.from_dict(document)) + assert result.status == "succeeded" and result.verification == "confirmed" + assert [(name, payload) for name, payload in server.writes if name != "update_topology"] == [ + ("assign", ("new", ("device:device1.dev.1",))), + ("unassign", ("old", ("device:device1.dev.1",))), + ] + delta = next(payload for name, payload in server.writes if name == "update_topology") + assert delta["replaceDevices"] == {} + assert all(vertex_id.startswith("device1.1.") for vertex_id in delta["replaceVertices"]) + assert server.vertex_forms["device1.1.vr.v"]["fields"]["label"] == "module-a-M1-RX-video-01" + assert inventory.writes == [] + + +def _mark_endpoint(server: FakeInspectServer, vertex_id: str) -> None: + for node in server.nodes.values(): + for module in node["modules"].values(): + for port in module["ports"].values(): + info = port["vertexInfo"] + sides = [info] if "id" in info else [side for side in (info.get("in"), info.get("out")) if side] + for side in sides: + if side.get("id") == vertex_id: + side["fields"]["isEndpoint"] = True + return + raise AssertionError(vertex_id) + + +def test_sibling_endpoint_label_change_rejects_the_plan( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + sibling = "device1.2.vs.v" + _mark_endpoint(server, sibling) + server.vertex_forms[sibling]["fields"]["label"] = "sibling-endpoint" + module = BlueprintDevice( + key="module-a", + label="module-a", + topology=ModuleTarget(device_id="device1", module_id="device1.dev.1"), + ) + document = { + "schema_version": 1, + "topology": {"default": {"vertex_processor": {"processor_type": "matrox.convertip.default"}}}, + } + plan = engine.plan(module, Blueprint.from_dict(document)) + assert plan.fully_resolved + server.vertex_forms[sibling]["fields"]["label"] = "renamed-endpoint" + with pytest.raises(BlueprintApplyError, match="changed since planning") as info: + plan.apply() + assert isinstance(info.value.__cause__, BlueprintConflictError) + assert server.writes == [] + + +def test_module_target_rejections(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + _chassis(server, inventory) + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + device_settings = {"schema_version": 1, "topology": {"default": {"device": {"icon_type": "monitor"}}}} + with pytest.raises(BlueprintTargetError, match="cannot be applied to a module target"): + engine.plan(module, Blueprint.from_dict(device_settings)) + missing = module.model_copy(update={"topology": ModuleTarget(device_id="device1", module_id="device2.dev.1")}) + with pytest.raises(BlueprintTargetError, match="does not belong"): + engine.plan(missing, Blueprint.from_dict({"schema_version": 1, "topology": {"default": {}}})) + + +def test_module_tag_failure_is_a_partial_result( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + server.fail_tag = "old" + document = { + "schema_version": 1, + "topology": { + "default": { + "module": {"tags": ["keep", "new"]}, + "vertex_processor": {"processor_type": "matrox.convertip.default"}, + } + }, + } + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + with pytest.raises(BlueprintApplyError) as info: + engine.apply(module, Blueprint.from_dict(document)) + result = info.value.result + assert result.status == "partial" + assert result.phase("topology").status == "completed" + assert result.phase("module_tags").status == "failed" + assert result.verification == "unconfirmed" + assert result.verification_detail is not None and "topology read-back did not run" in result.verification_detail + assert [op.action for op in result.phase("module_tags").operations] == ["assign_tag"] + + +# --- Naming, overrides, mapping, custom processors --- + + +def test_label_collisions_and_overrides( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + colliding = NamingScheme(endpoint_label="{device.label}-{endpoint.direction}") + with pytest.raises(BlueprintValidationError, match="Endpoint label collision"): + engine.plan(_device(), _blueprint(), naming=colliding) + assert engine.plan( + _device(), _blueprint(), naming=colliding, options=ApplyOptions(naming_collisions="allow") + ).has_changes + + overrides = [{"factory_label": "Audio Sender", "direction": "In", "fields": {"label": "custom-audio"}}] + plan = engine.plan(_device(), _blueprint(vertices=overrides), naming=colliding) + changes = {op.entity_id: {c.field: c for c in op.changes} for op in plan.phase("topology").operations} + assert changes["device1.0.as.v"]["label"].after == "custom-audio" + assert changes["device1.0.as.v"]["label"].source.startswith("override:") + + with pytest.raises(BlueprintTargetError, match="must match exactly one vertex"): + engine.plan(_device(), _blueprint(vertices=[{"factory_label": "P1", "fields": {"active": False}}])) + + +def test_interface_mapping_rules(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + _existing(inventory, server) + mapping = {"a": ["X{module.position}", "P{module.position}"], "b": ["P2"]} + plan = engine.plan(_device(module_position="1"), _blueprint(ip_vertex_mapping=mapping)) + assert [(b.key, b.candidate, b.out_vertex_id) for b in plan.interface_bindings] == [ + ("a", "P1", "device1.0.P1.out"), + ("b", "P2", "device1.0.P2.out"), + ] + with pytest.raises(BlueprintTargetError, match="no IP port .* matches 'X1', 'X2'"): + engine.plan(_device(), _blueprint(ip_vertex_mapping={"a": ["X1", "X2"]})) + with pytest.raises(BlueprintValidationError, match="requires BlueprintDevice.module_position"): + engine.plan(_device(), _blueprint(ip_vertex_mapping={"a": ["P{module.position}"]})) + + +def test_custom_processor_and_naming_change_rename_the_same_ids( + inventory: FakeInventory, server: FakeInspectServer, app: Any +) -> None: + _existing(inventory, server) + engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) + document = { + "schema_version": 1, + "topology": { + "default": { + "vertex_processor": { + "processor_type": "example-org.simple-video", + "params": {"factory_label": "Video Sender"}, + } + } + }, + } + plan = engine.plan(_device(), Blueprint.from_dict(document)) + engine.register_processor("example-org.later", SimpleVideoProcessor) # does not affect the existing plan + engine.configure(naming=NamingScheme(endpoint_label="{device.label}-later")) + plan.apply() + assert server.vertex_forms["device1.0.vs.v"]["fields"]["label"] == "device-a-TX-video-01" + + engine.apply(_device(), Blueprint.from_dict(document), naming=NamingScheme(endpoint_label="{device.label}-video")) + assert server.vertex_forms["device1.0.vs.v"]["fields"]["label"] == "device-a-video" + + +def test_topology_only_virtual_device(engine: BlueprintEngine, server: FakeInspectServer) -> None: + layout = matrox_layout("virtual.1", "tx") + layout["node"] = {"_id": "virtual-1", "modules": {}} + server.install(layout) + server.nodes["virtual-1"] = server.nodes.pop("virtual.1") + device = BlueprintDevice(key="virtual-a", label="virtual-a", topology=DeviceTarget(device_id="virtual.1")) + document = {"schema_version": 1, "topology": {"default": {"device": {"icon_type": "server"}}}} + result = engine.apply(device, Blueprint.from_dict(document)) + assert result.status == "succeeded" + assert server.device_forms["virtual.1"]["iconType"] == "server" + assert server.device_forms["virtual.1"]["descriptor"]["label"] == "virtual-a" + + +def test_from_inventory_and_engine_validation(engine: BlueprintEngine, inventory: FakeInventory) -> None: + record = inventory.seed("device1", label="device-a") + device = BlueprintDevice.from_inventory(record) + assert (device.key, device.label, device.inventory_id, device.management_address) == ( + "device1", + "device-a", + "device1", + None, + ) + engine.validate(_blueprint()) + unknown = {"schema_version": 1, "topology": {"default": {"vertex_processor": {"processor_type": "example-org.x"}}}} + with pytest.raises(BlueprintValidationError, match="not registered"): + engine.validate(Blueprint.from_dict(unknown)) + + +class _ModeParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + mode: str + + +class MisbehavingProcessor(VertexProcessor[_ModeParams]): + """Returns deliberately invalid output to exercise the engine's output validation.""" + + params_model = _ModeParams + + def process(self, context: ProcessingContext, params: _ModeParams) -> Any: + codec = context.find_vertices(kind="codec")[0] + if params.mode == "sibling": + return ProcessorResult(vertices=[VertexEdit(vertex_id="device1.2.vs.v", fields=VertexPatch(active=False))]) + if params.mode == "device-patch": + return ProcessorResult(device={"icon_type": "monitor"}) + if params.mode == "conflict": + return ProcessorResult( + vertices=[ + VertexEdit(vertex_id=codec.id, fields=VertexPatch(label="a")), + VertexEdit(vertex_id=codec.id, fields=VertexPatch(label="b")), + ] + ) + if params.mode == "duplicate": + edit = VertexEdit(vertex_id=codec.id, fields=VertexPatch(active=True)) + return ProcessorResult(vertices=[edit, edit]) + if params.mode == "bypass": + return ProcessorResult.model_construct( + vertices=[VertexEdit.model_construct(vertex_id=codec.id, fields={"label": 5})] + ) + if params.mode == "sdp-on-ip": + ip = context.find_vertices(kind="ip")[0] + return ProcessorResult(vertices=[VertexEdit(vertex_id=ip.id, fields=VertexPatch(sdp_support=True))]) + if params.mode == "wrong-type": + return {"vertices": []} + raise RuntimeError("boom") + + +@pytest.mark.parametrize( + ("mode", "error", "message"), + [ + ("sibling", BlueprintTargetError, "outside module 'device1.dev.1'"), + ("device-patch", BlueprintTargetError, "device-wide changes for a module target"), + ("conflict", ProcessorInputError, "conflicting 'label' values"), + ("bypass", ProcessorInputError, "invalid result"), + ("sdp-on-ip", BlueprintCapabilityError, "only supported on codec vertices"), + ("wrong-type", ProcessorInputError, "must return a ProcessorResult"), + ("crash", ProcessorInputError, "failed: RuntimeError: boom"), + ], +) +def test_processor_output_is_validated( + inventory: FakeInventory, server: FakeInspectServer, app: Any, mode: str, error: type, message: str +) -> None: + _chassis(server, inventory) + engine = BlueprintEngine(app, processors={"example-org.bad": MisbehavingProcessor}) + document = { + "schema_version": 1, + "topology": {"default": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": mode}}}}, + } + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + with pytest.raises(error, match=message): + engine.plan(module, Blueprint.from_dict(document)) + assert server.writes == [] + + +def test_identical_duplicate_edits_are_deduplicated( + inventory: FakeInventory, server: FakeInspectServer, app: Any +) -> None: + _chassis(server, inventory) + engine = BlueprintEngine(app, processors={"example-org.bad": MisbehavingProcessor}) + document = { + "schema_version": 1, + "topology": { + "default": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": "duplicate"}}} + }, + } + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + assert engine.plan(module, Blueprint.from_dict(document)).phase("topology").status == "no_change" + + +def test_module_tag_baseline_conflict_after_topology_commit( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + document = { + "schema_version": 1, + "topology": { + "default": { + "module": {"tags": ["keep", "new"]}, + "vertex_processor": {"processor_type": "matrox.convertip.default"}, + } + }, + } + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + plan = engine.plan(module, Blueprint.from_dict(document)) + original_commit = server.update_topology + + def commit_then_retag(delta: Any) -> Any: + response = original_commit(delta) + server.nodes["device1"]["modules"]["device1.dev.1"]["tagsInfo"]["assigned"]["local"]["other"] = {} + return response + + server.update_topology = commit_then_retag # type: ignore[method-assign] + with pytest.raises(BlueprintApplyError) as info: + plan.apply() + result = info.value.result + assert (result.status, result.phase("topology").status, result.phase("module_tags").status) == ( + "partial", + "completed", + "failed", + ) + assert not [name for name, _ in server.writes if name in ("assign", "unassign")] + + +def test_module_tags_without_local_distinction_allow_only_additions( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + server.nodes["device1"]["modules"]["device1.dev.1"]["tagsInfo"] = {"assigned": {"all": ["inherited"]}} + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + + def plan_with(tags: Any) -> Any: + return engine.plan( + module, Blueprint.from_dict({"schema_version": 1, "topology": {"default": {"module": {"tags": tags}}}}) + ) + + with pytest.raises(BlueprintCapabilityError, match="cannot be distinguished"): + plan_with(["keep"]) + with pytest.raises(BlueprintCapabilityError, match="cannot be distinguished"): + plan_with({"remove": ["inherited"]}) + assert [op.action for op in plan_with({"add": ["new"]}).phase("module_tags").operations] == ["assign_tag"] + + +def test_module_with_own_inventory_record_never_touches_the_parent( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + inventory.seed("device5", label="module-a", address="192.0.2.55", custom={"port": 80}) + server.sync["device1"] = {"add": {"x": 1}, "update": {}, "remove": {}} + module = BlueprintDevice( + key="module-a", + label="module-a", + inventory_id="device5", + topology=ModuleTarget(device_id="device1", module_id="device1.dev.1"), + ) + plan = engine.plan(module, _blueprint(device=None)) + assert plan.phase("inventory").status == "planned" and plan.phase("topology").status == "deferred" + result = plan.apply() + assert result.status == "succeeded" and result.materialized + assert inventory.writes == [("update", "device5")] + assert not [name for name, _ in server.writes if name in ("add_devices", "sync_devices")] + delta = next(payload for name, payload in server.writes if name == "update_topology") + assert delta["replaceDevices"] == {} and all(v.startswith("device1.1.") for v in delta["replaceVertices"]) + + +# --- Dry run --- + + +def test_dry_run_checks_everything_but_writes_nothing( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + plan = engine.plan(_device(), _blueprint()) + + preview = plan.apply(dry_run=True) + assert (preview.status, preview.dry_run, preview.ok, preview.verification) == ( + "planned", + True, + True, + "not_applicable", + ) + assert [(p.name, p.status) for p in preview.phases] == [ + ("inventory", "planned"), + ("discovery", "skipped"), + ("topology_sync", "no_change"), + ("topology", "planned"), + ("module_tags", "skipped"), + ("verification", "skipped"), + ] + assert preview.phase("topology").operations == plan.phase("topology").operations + assert [b.key for b in preview.interface_bindings] == ["stream-a", "stream-b"] + assert inventory.writes == [] and server.writes == [] and app.inspect._snapshot is None + + result = plan.apply() # the same plan still applies after a dry run + assert result.status == "succeeded" and not result.dry_run + assert engine.apply(_device(), _blueprint(), dry_run=True).status == "no_change" + + +def test_dry_run_detects_stale_plans( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + plan = engine.plan(_device(), _blueprint()) + server.vertex_forms["device1.0.vs.v"]["fields"]["desc"] = "edited-elsewhere" + with pytest.raises(BlueprintApplyError) as info: + plan.apply(dry_run=True) + assert info.value.result.status == "failed" and info.value.result.dry_run + assert server.writes == [] + + +def test_dry_run_reports_deferred_work_for_creation( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + result = engine.apply(device, _blueprint(), dry_run=True) + assert result.status == "planned" and result.inventory_id is None + assert result.phase("inventory").operations[0].action == "create" + assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { + "deferred" + } + assert inventory.writes == [] and server.writes == [] + + +def test_dry_run_module_tags(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: + _chassis(server, inventory) + document = {"schema_version": 1, "topology": {"default": {"module": {"tags": ["keep", "new"]}}}} + module = BlueprintDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + result = engine.apply(module, Blueprint.from_dict(document), dry_run=True) + assert result.phase("module_tags").status == "planned" + assert [op.action for op in result.phase("module_tags").operations] == ["assign_tag", "unassign_tag"] + assert server.writes == [] + + +class _KindFieldParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + ports_on_ip: bool = False + + +class KindFieldProcessor(VertexProcessor[_KindFieldParams]): + """Proposes destination ports (codec) and static IGMP support (IP).""" + + params_model = _KindFieldParams + + def process(self, context: ProcessingContext, params: _KindFieldParams) -> ProcessorResult: + codec = context.require_vertex(kind="codec", factory_label="Video Sender") + ip = context.require_vertex(kind="ip", factory_label="P1", vertex_type="Out") + port_target = ip if params.ports_on_ip else codec + return ProcessorResult( + vertices=[ + VertexEdit( + vertex_id=port_target.id, + fields=VertexPatch(main_destination_port=50311, spare_destination_port=50312), + ), + VertexEdit(vertex_id=ip.id, fields=VertexPatch(supports_static_igmp=True)), + ] + ) + + +def test_kind_specific_fields_round_trip(inventory: FakeInventory, server: FakeInspectServer, app: Any) -> None: + _existing(inventory, server) + engine = BlueprintEngine(app, processors={"example-org.kind-fields": KindFieldProcessor}) + document = { + "schema_version": 1, + "topology": {"default": {"vertex_processor": {"processor_type": "example-org.kind-fields"}}}, + } + result = engine.apply(_device(), Blueprint.from_dict(document), scope="topology") + assert result.status == "succeeded" and result.verification == "confirmed" + generic = server.vertex_forms["device1.0.vs.v"]["fields"]["typeFields"]["generic"] + assert (generic["mainDstInfo"]["port"], generic["spareDstInfo"]["port"]) == (50311, 50312) + assert server.vertex_forms["device1.0.P1.out"]["fields"]["typeFields"]["supportsStaticIgmpCfg"] is True + assert ( + engine.plan(_device(), Blueprint.from_dict(document), scope="topology").phase("topology").status == "no_change" + ) + + wrong = { + "schema_version": 1, + "topology": { + "default": { + "vertex_processor": {"processor_type": "example-org.kind-fields", "params": {"ports_on_ip": True}} + } + }, + } + with pytest.raises(BlueprintCapabilityError, match="'main_destination_port' is only supported on codec vertices"): + engine.plan(_device(), Blueprint.from_dict(wrong), scope="topology") + + +def test_engine_schema_lists_only_registered_processors(app: Any) -> None: + engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) + spec = engine.json_schema()["$defs"]["VertexProcessorSpec"] + assert spec["properties"]["processor_type"]["enum"] == ["example-org.simple-video", "matrox.convertip.default"] + assert "enum" not in published_json_schema()["$defs"]["VertexProcessorSpec"]["properties"]["processor_type"] diff --git a/tests/blueprints/test_loader.py b/tests/blueprints/test_loader.py new file mode 100644 index 0000000..f5b4130 --- /dev/null +++ b/tests/blueprints/test_loader.py @@ -0,0 +1,86 @@ +"""Strict YAML loading: safety rules, scalar resolution, and source locations.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from videoipath_automation_tool.blueprints import Blueprint, BlueprintValidationError +from videoipath_automation_tool.blueprints.loader import MAX_DOCUMENT_BYTES, parse_yaml + +MINIMAL = "schema_version: 1\ntopology:\n default: {}\n" + + +def _issue(text: str | bytes) -> str: + with pytest.raises(BlueprintValidationError) as info: + parse_yaml(text, source="bp.yml") + return str(info.value) + + +def test_scalars_are_resolved_strictly() -> None: + data, _ = parse_yaml("a: true\nb: off\nc: yes\nd: 2024-01-01\ne: 010\nf: 1.5\ng: ~\nh: 07:30\ni: -3\n") + assert data == { + "a": True, + "b": "off", + "c": "yes", + "d": "2024-01-01", + "e": "010", + "f": 1.5, + "g": None, + "h": "07:30", + "i": -3, + } + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("a: 1\na: 2\n", "duplicate key 'a'"), + ("a:\n b:\n c: 1\n c: 2\n", "duplicate key 'c'"), + ("a: &anchor 1\n", "anchors are not supported"), + ("<<: {a: 1}\n", "merge keys"), + ("a: !custom 1\n", "unsupported scalar tag"), + ("a: !!python/object:os.system x\n", "unsupported scalar tag"), + ("- 1\n- 2\n", "root must be a mapping"), + ("a: 1\n---\nb: 2\n", "another document"), + ("1: one\n", "mapping keys must be strings"), + ("? [a]\n: 1\n", "mapping keys must be strings"), + ("", "document is empty"), + ("a: [1\n", "invalid YAML"), + ], +) +def test_rejected_documents(text: str, message: str) -> None: + assert message in _issue(text) + + +def test_aliases_are_rejected() -> None: + assert "anchors are not supported" in _issue("a: &x [1]\nb: *x\n") + + +def test_size_depth_and_encoding_limits() -> None: + assert "exceeds" in _issue(b"a: " + b"x" * MAX_DOCUMENT_BYTES) + assert "depth" in _issue("a: " + "[" * 40 + "]" * 40 + "\n") + assert "UTF-8" in _issue(b"a: \xff\xfe\n") + + +def test_errors_carry_source_line_and_column() -> None: + text = "schema_version: 1\ntopology:\n default:\n vertex_processor:\n processor_type: x\n params:\n flag: 1\n extra: 2\n" + with pytest.raises(BlueprintValidationError) as info: + Blueprint.from_yaml(text, source="bp.yml") + issue = info.value.issues[0] + assert (issue.source, issue.line, issue.column, issue.path) == ( + "bp.yml", + 8, + 14, + "topology.default.vertex_processor.extra", + ) + assert str(issue).startswith("bp.yml:8:14\ntopology.default.vertex_processor.extra\n") + + +def test_load_from_file_and_bytes(tmp_path: Path) -> None: + path = tmp_path / "bp.yml" + path.write_text(MINIMAL, encoding="utf-8") + assert Blueprint.load(path).variant_names("topology") == ["default"] + assert Blueprint.from_yaml(MINIMAL.encode("utf-8")).topology is not None + assert Blueprint.from_yaml("\ufeff" + MINIMAL).topology is not None diff --git a/tests/blueprints/test_naming.py b/tests/blueprints/test_naming.py new file mode 100644 index 0000000..21aef80 --- /dev/null +++ b/tests/blueprints/test_naming.py @@ -0,0 +1,125 @@ +"""Naming blocks, templates, layering, and rendering errors.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from videoipath_automation_tool.blueprints import ( + DEFAULT_NAMING, + BlueprintValidationError, + Field, + Join, + NameContext, + NamingScheme, + Text, +) +from videoipath_automation_tool.blueprints.naming import BlueprintNaming, render_name + + +def _context(position: str | None = None, index: int = 1, **attributes: object) -> NameContext: + return NameContext( + device={"key": "key-a", "label": "device-a", "description": None}, + module={"position": position}, + endpoint={"direction": "TX", "media": "video", "index": index, "engine": 0, "program": 1, "leg": None}, + vertex={"factory_label": "Video Sender", "id": "device1.0.vs.v"}, + attributes=dict(attributes), + ) + + +def test_default_endpoint_label_with_optional_module() -> None: + expression = DEFAULT_NAMING.endpoint_label + assert render_name("endpoint_label", expression, _context()) == "device-a-TX-video-01" + assert render_name("endpoint_label", expression, _context(position="0")) == "device-a-M0-TX-video-01" + assert render_name("endpoint_label", expression, _context(position="A1", index=123)) == "device-a-MA1-TX-video-123" + + +def test_engine_program_media_code_convention() -> None: + scheme = NamingScheme( + endpoint_label=Join( + parts=[ + Field("device.label"), + "E{endpoint.engine}P{endpoint.program}", + Join( + parts=[Field("endpoint.direction"), Field("endpoint.media", mapping={"video": "20", "audio": "30"})] + ), + Field("endpoint.index", format="02d"), + ], + separator="-", + ) + ) + assert render_name("endpoint_label", scheme.endpoint_label, _context()) == "device-a-E0P1-TX20-01" + + +def test_yaml_structured_form_and_text_block() -> None: + naming = BlueprintNaming.model_validate( + { + "endpoint_label": { + "join": ["{device.label}", {"field": "attributes.site", "suffix": "!"}, {"text": "x"}], + "separator": "_", + } + } + ) + assert render_name("endpoint_label", naming.endpoint_label, _context(site="site-ä")) == "device-a_site-ä!_x" + assert render_name("device_label", Text("fixed"), _context()) == "fixed" + assert render_name("device_label", "{{literal}}-{device.label}", _context()) == "{literal}-device-a" + + +@pytest.mark.parametrize( + ("entry", "expression", "message"), + [ + ("inventory_label", "{device.label!r}", "invalid field reference"), + ("inventory_label", "{device.label[0]}", "invalid field reference"), + ("inventory_label", "{device.nope}", "unknown field"), + ("inventory_label", "{endpoint.index}", "not available for this entry"), + ("endpoint_label", "{endpoint.index:x}", "unsupported format"), + ("endpoint_label", "{endpoint.index:{width}}", "nested braces"), + ("endpoint_label", "{device.label", "unclosed"), + ("endpoint_label", "a}b", "single '}'"), + ], +) +def test_static_validation(entry: str, expression: str, message: str) -> None: + with pytest.raises(ValidationError, match=message): + NamingScheme(**{entry: expression}) + + +@pytest.mark.parametrize( + ("expression", "code"), + [ + ("{device.description}", "naming.missing"), + (Join(parts=["{device.description}"], skip_missing=False), "naming.missing"), + (Field("endpoint.media", mapping={"audio": "30"}), "naming.render"), + (Field("device.label", format="02d"), "naming.render"), + (Field("attributes.nested"), "naming.render"), + (Text(" "), "naming.empty"), + (Text("a\nb"), "naming.invalid"), + ], +) +def test_render_errors(expression: object, code: str) -> None: + with pytest.raises(BlueprintValidationError) as info: + render_name("endpoint_label", expression, _context(nested={"a": 1})) + assert info.value.codes == [code] + + +def test_skip_missing_only_skips_absent_values() -> None: + join = Join( + parts=["{device.label}", "{device.description}", "{attributes.missing}"], separator="-", skip_missing=True + ) + assert render_name("endpoint_label", join, _context()) == "device-a" + failing = Join(parts=["{device.label}", Field("endpoint.media", mapping={"audio": "30"})], skip_missing=True) + with pytest.raises(BlueprintValidationError, match="no mapping"): + render_name("endpoint_label", failing, _context()) + + +def test_python_renderer_and_layering() -> None: + class Upper: + def render(self, context: NameContext) -> str: + return str(context.value("device.label")).upper() + + engine = NamingScheme(device_label=Upper()) + call = NamingScheme(endpoint_label=None) + layered = NamingScheme.layered(DEFAULT_NAMING, engine, None, call) + assert render_name("device_label", layered.device_label, _context()) == "DEVICE-A" + assert layered.endpoint_label is None + assert layered.inventory_label == "{device.label}" + assert set(layered.entries()) == {"inventory_label", "device_label", "endpoint_label"} diff --git a/tests/blueprints/test_processors.py b/tests/blueprints/test_processors.py new file mode 100644 index 0000000..5831cb6 --- /dev/null +++ b/tests/blueprints/test_processors.py @@ -0,0 +1,285 @@ +"""Processor contract, registry, detached context, and the built-in Matrox ConvertIP processor.""" + +from __future__ import annotations + +import random +from typing import Any + +import pytest +from pydantic import BaseModel, ConfigDict, ValidationError + +from tests.blueprints.conftest import FakeInspectServer, make_inspect_app, matrox_layout +from videoipath_automation_tool.blueprints import ( + BlueprintCapabilityError, + BlueprintValidationError, + DeviceTarget, + InterfaceBinding, + MatroxConvertIPParams, + MatroxConvertIPProcessor, + ModuleTarget, + ProcessingContext, + ProcessorInputError, + ProcessorRegistry, + ProcessorResult, + SourceFacts, + TagDelta, + VertexProcessor, +) +from videoipath_automation_tool.blueprints.inspect import InspectGateway, build_context + + +def _context( + layout: dict[str, Any], + *, + module: str | None = None, + description: str | None = None, + interfaces: tuple[InterfaceBinding, ...] = (), + mutate: Any = None, +) -> ProcessingContext: + server = FakeInspectServer() + server.install(layout) + if mutate is not None: + mutate(server) + target = ( + ModuleTarget(device_id=layout["device_id"], module_id=module) + if module + else DeviceTarget(device_id=layout["device_id"]) + ) + scope = InspectGateway(make_inspect_app(server)).read_scope(target) + source = SourceFacts(key="key-a", label="device-a", description=description) + return build_context(scope, source, list(interfaces), None, None) + + +def _process(context: ProcessingContext, **params: Any) -> ProcessorResult: + return MatroxConvertIPProcessor().process(context, MatroxConvertIPParams(**params)) + + +def _by_label(result: ProcessorResult, context: ProcessingContext) -> dict[str, Any]: + return {context.vertex(edit.vertex_id).factory_label: edit for edit in result.vertices} + + +# --- Registry --- + + +class _Params(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class _Noop(VertexProcessor[_Params]): + params_model = _Params + + def process(self, context: ProcessingContext, params: _Params) -> ProcessorResult: + return ProcessorResult() + + +def test_registry_rules() -> None: + mapping = {"example-org.noop": _Noop} + registry = ProcessorRegistry(mapping) + mapping["example-org.other"] = _Noop + assert registry.ids() == ["example-org.noop", "matrox.convertip.default"] + + with pytest.raises(ValueError, match="already registered"): + registry.register("matrox.convertip.default", _Noop) + with pytest.raises(ValueError, match="Invalid processor id"): + registry.register("bad id", _Noop) + with pytest.raises(TypeError, match="VertexProcessor subclass"): + registry.register("example-org.x", object) # type: ignore[arg-type] + with pytest.raises(BlueprintValidationError, match="Available: example-org.noop, matrox.convertip.default"): + registry.get("missing") + + clone = registry.copy() + clone.register("example-org.clone", _Noop) + assert "example-org.clone" not in registry + assert registry.params_schema("matrox.convertip.default")["title"] == "MatroxConvertIPParams" + + +def test_params_validation_carries_paths() -> None: + with pytest.raises(BlueprintValidationError) as info: + ProcessorRegistry().validate_params("matrox.convertip.default", {"mode": "sideways"}, ("params",)) + assert info.value.issues[0].path == "params.mode" + + +# --- Context --- + + +def test_context_is_detached_scoped_and_immutable() -> None: + layout = matrox_layout("device1", "rx") + context = _context(layout) + with pytest.raises(ValidationError): + context.vertices[0].label = "changed" # type: ignore[misc] + assert len(context.find_vertices(kind="codec")) == 2 + assert context.require_vertex(factory_label="Video Receiver", vertex_type="Out").kind == "codec" + with pytest.raises(ProcessorInputError, match="found 6 vertices"): + context.require_vertex(kind="ip") + + +def test_module_context_contains_only_module_vertices() -> None: + layout = matrox_layout("device1", "rx", module="device1.dev.1") + sibling = matrox_layout("device1", "tx", module="device1.dev.2") + layout["node"]["modules"].update(sibling["node"]["modules"]) + layout["vertex_forms"].update(sibling["vertex_forms"]) + context = _context(layout, module="device1.dev.2") + assert context.scope == "module" + assert {v.module_id for v in context.vertices} == {"device1.dev.2"} + assert {v.vertex_type for v in context.find_vertices(kind="codec")} == {"In"} + + +# --- Matrox recognition --- + + +def test_tx_layout() -> None: + context = _context(matrox_layout("device1", "tx"), description="studio-a") + result = _process(context, video_sender_tags=["video-tag-b"]) + edits = _by_label(result, context) + video = edits["Video Sender"] + assert video.endpoint.model_dump() == { + "direction": "TX", + "media": "video", + "index": 1, + "engine": 0, + "program": 1, + "leg": None, + } + assert video.fields.managed() == { + "use_as_endpoint": True, + "description": "Video Sender | studio-a", + "sdp_support": True, + "tags": ["video-tag-b"], + } + assert "tags" not in edits["Audio Sender"].fields.managed() + assert result.device.icon_type == "gateway" + + +def test_rx_layout_without_sdp_and_with_sips() -> None: + context = _context(matrox_layout("device1", "rx")) + result = _process(context, redundant_streams=False, audio_receiver_tags=[]) + audio = _by_label(result, context)["Audio Receiver"] + assert audio.fields.managed() == { + "use_as_endpoint": True, + "description": "Audio Receiver", + "sips_mode": "NONE", + "tags": [], + } + assert result.device.icon_type == "monitor" + + +def test_four_split_layout_normalizes_indices() -> None: + context = _context(matrox_layout("device1", "four_split")) + result = _process(context, mode="four_split", video_receiver_tags=TagDelta(add=["video-tag-c"])) + edits = _by_label(result, context) + assert [edits[f"Video Receiver {i}"].endpoint.index for i in range(4)] == [1, 2, 3, 4] + assert edits["Audio Receiver"].endpoint.index == 1 + assert edits["Video Receiver 0"].fields.tags == TagDelta(add=["video-tag-c"]) + + +def test_processing_is_deterministic_regardless_of_order() -> None: + context = _context(matrox_layout("device1", "four_split")) + shuffled = list(context.vertices) + random.Random(4).shuffle(shuffled) + other = context.model_copy(update={"vertices": tuple(shuffled)}) + assert _process(context) == _process(other) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda s: s.vertex_forms.pop("device1.0.ar.v"), "unknown kind"), + (lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver 7"), "indices"), + (lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver"), "no 'Video Receiver ' index"), + (lambda s: _set_media(s, "device1.0.ar.v", "video"), "unsupported RX layout with 5 video and 0 audio"), + ], +) +def test_invalid_four_split_layouts(mutate: Any, message: str) -> None: + context = _context(matrox_layout("device1", "four_split"), mutate=mutate) + with pytest.raises(ProcessorInputError, match=message): + _process(context) + + +def test_mode_must_agree_with_observed_layout() -> None: + with pytest.raises(ProcessorInputError, match="mode 'rx' was requested"): + _process(_context(matrox_layout("device1", "tx")), mode="rx") + + +def test_mixed_directions_and_missing_codecs_fail() -> None: + def mixed(server: FakeInspectServer) -> None: + node_port = server.nodes["device1"]["modules"]["device1.dev.0"]["ports"]["device1.dev.0.as"] + node_port["vertexInfo"]["vertexType"] = "Out" + + with pytest.raises(ProcessorInputError, match="all be 'In'"): + _process(_context(matrox_layout("device1", "tx"), mutate=mixed)) + + def no_codecs(server: FakeInspectServer) -> None: + for form in server.vertex_forms.values(): + form["fields"]["typeFields"]["type"] = "ip" + + with pytest.raises(ProcessorInputError, match="no codec vertices"): + _process(_context(matrox_layout("device1", "tx"), mutate=no_codecs)) + + +def test_label_fallback_and_null_audio_workaround() -> None: + def unstructured(label: str) -> Any: + def mutate(server: FakeInspectServer) -> None: + for vertex_id in ("device1.0.vs.v", "device1.0.as.v"): + type_fields = server.vertex_forms[vertex_id]["fields"]["typeFields"] + type_fields["generic"]["codecFormat"] = None + type_fields["specific"]["type"] = None + _relabel(server, "device1.0.as.v", label) + + return mutate + + context = _context(matrox_layout("device1", "tx"), mutate=unstructured("Main Audio")) + assert _by_label(_process(context), context)["Main Audio"].endpoint.media == "audio" + + context = _context(matrox_layout("device1", "tx"), mutate=unstructured("null")) + with pytest.raises(ProcessorInputError, match="cannot classify"): + _process(context) + result = _process(context, allow_null_audio_label=True) + assert _by_label(result, context)["null"].endpoint.media == "audio" + assert any(d.code == "matrox.null_audio_label" for d in result.diagnostics) + + context = _context(matrox_layout("device1", "tx"), mutate=unstructured("null-ish")) + with pytest.raises(ProcessorInputError, match="cannot classify"): + _process(context, allow_null_audio_label=True) + + +def test_redundancy_validation() -> None: + context = _context(matrox_layout("device1", "tx")) + with pytest.raises(ProcessorInputError, match="exactly two 'Out' streaming IP vertices"): + _process(context, redundant_streams=True) # MGMT counts without an explicit mapping + bindings = ( + InterfaceBinding( + key="a", candidate="P1", port_id="p1", in_vertex_id="device1.0.P1.in", out_vertex_id="device1.0.P1.out" + ), + InterfaceBinding( + key="b", candidate="P2", port_id="p2", in_vertex_id="device1.0.P2.in", out_vertex_id="device1.0.P2.out" + ), + ) + result = _process(_context(matrox_layout("device1", "tx"), interfaces=bindings), redundant_streams=True) + assert any("device1.0.P1.out, device1.0.P2.out" in d.message for d in result.diagnostics) + + +def test_control_configuration_is_an_explicit_capability_error() -> None: + with pytest.raises(BlueprintCapabilityError, match="control"): + _process(_context(matrox_layout("device1", "tx")), configure_control=True) + + +def test_module_target_proposes_no_device_patch() -> None: + layout = matrox_layout("device1", "rx", module="device1.dev.3") + result = _process(_context(layout, module="device1.dev.3")) + assert result.device is None + + +# --- Helpers --- + + +def _relabel(server: FakeInspectServer, vertex_id: str, label: str) -> None: + for module in server.nodes["device1"]["modules"].values(): + for port in module["ports"].values(): + if port["vertexInfo"].get("id") == vertex_id: + port["label"] = label + + +def _set_media(server: FakeInspectServer, vertex_id: str, media: str) -> None: + type_fields = server.vertex_forms[vertex_id]["fields"]["typeFields"] + type_fields["generic"]["codecFormat"] = media.capitalize() + type_fields["specific"]["type"] = media diff --git a/tests/blueprints/test_resolution.py b/tests/blueprints/test_resolution.py new file mode 100644 index 0000000..216e0da --- /dev/null +++ b/tests/blueprints/test_resolution.py @@ -0,0 +1,241 @@ +"""Document validation, variant resolution, driver validation, and schema export.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest +from pydantic import BaseModel, ConfigDict + +from videoipath_automation_tool.blueprints import ( + Blueprint, + BlueprintValidationError, + InventorySettings, + ProcessorRegistry, + ProcessorResult, + VertexProcessor, + published_json_schema, +) +from videoipath_automation_tool.blueprints.resolution import resolve_blueprint + +NMOS = "com.nevion.NMOS_multidevice-0.1.0" + + +def _document() -> dict[str, Any]: + return { + "schema_version": 1, + "inventory": { + "default": { + "driver_id": NMOS, + "generic_settings": {"enable_https": False, "http_auth": 0}, + "custom_settings": {"port": 8080, "disable_rx_sdp_with_null": True}, + "metadata": {"site": "site-a"}, + }, + "secure": {"generic_settings": {"enable_https": True}, "custom_settings": {"port": 443}}, + "other-driver": {"driver_id": "com.nevion.NMOS-0.1.0", "custom_settings": {"always_enable_rtp": True}}, + }, + "topology": { + "default": { + "device": {"icon_size": "medium", "tags": ["tag-a"]}, + "vertex_processor": { + "processor_type": "matrox.convertip.default", + "params": {"redundant_streams": True, "video_receiver_tags": ["video-tag-a"]}, + }, + "naming": {"endpoint_label": "{device.label}-{endpoint.index:02d}"}, + }, + "receiver": {"vertex_processor": {"params": {"mode": "rx", "video_receiver_tags": []}}}, + "no-processor": {"vertex_processor": None, "device": {"tags": []}}, + "custom": {"vertex_processor": {"processor_type": "example-org.simple", "params": {"factory_label": "x"}}}, + }, + } + + +class _SimpleParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + factory_label: str + + +class _Simple(VertexProcessor[_SimpleParams]): + params_model = _SimpleParams + + def process(self, context: Any, params: _SimpleParams) -> ProcessorResult: + return ProcessorResult() + + +def _registry() -> ProcessorRegistry: + return ProcessorRegistry({"example-org.simple": _Simple}) + + +def _resolve(blueprint: Blueprint, **kwargs: Any) -> Any: + options = {"scope": "all", "inventory_variant": "default", "topology_variant": "default", "registry": _registry()} + options.update(kwargs) + return resolve_blueprint(blueprint, **options) + + +def test_named_variant_patches_default_and_records_provenance() -> None: + resolved = _resolve(Blueprint.from_dict(_document()), inventory_variant="secure", topology_variant="receiver") + assert resolved.inventory.custom_settings == {"port": 443, "disable_rx_sdp_with_null": True} + assert resolved.inventory.config.generic_settings.managed() == {"enable_https": True, "http_auth": 0} + assert resolved.inventory.provenance["custom_settings.port"] == "variant:secure" + assert resolved.inventory.provenance["custom_settings.disable_rx_sdp_with_null"] == "default" + params = resolved.topology.params + assert (params.mode, params.redundant_streams, params.video_receiver_tags) == ("rx", True, []) + assert resolved.naming.entries() == {"endpoint_label": "{device.label}-{endpoint.index:02d}"} + + +def test_null_disables_processor_and_empty_list_replaces() -> None: + resolved = _resolve(Blueprint.from_dict(_document()), topology_variant="no-processor") + assert resolved.topology.processor_cls is None + assert resolved.topology.config.device.managed() == {"icon_size": "medium", "tags": []} + + +def test_processor_change_does_not_inherit_parameters() -> None: + resolved = _resolve(Blueprint.from_dict(_document()), topology_variant="custom") + assert resolved.topology.processor_id == "example-org.simple" + assert resolved.topology.params == _SimpleParams(factory_label="x") + + +def test_driver_change_replaces_custom_settings_base() -> None: + resolved = _resolve(Blueprint.from_dict(_document()), inventory_variant="other-driver") + assert resolved.inventory.driver_id == "com.nevion.NMOS-0.1.0" + assert resolved.inventory.custom_settings == {"always_enable_rtp": True} + assert resolved.inventory.config.metadata == {"site": "site-a"} + + +def test_instance_overrides_apply_last() -> None: + overrides = InventorySettings(custom_settings={"port": 9000}, active=False) + resolved = _resolve(Blueprint.from_dict(_document()), overrides=overrides) + assert resolved.inventory.custom_settings["port"] == 9000 + assert resolved.inventory.provenance["custom_settings.port"] == "instance" + assert resolved.inventory.config.active is False + + +def test_unknown_variant_never_falls_back() -> None: + with pytest.raises(BlueprintValidationError, match="Available: default, receiver, no-processor, custom"): + _resolve(Blueprint.from_dict(_document()), topology_variant="reciever") + + +def test_section_selection_rules() -> None: + topology_only = Blueprint.from_dict({"schema_version": 1, "topology": {"default": {}}}) + assert _resolve(topology_only).skipped == {"inventory": "section absent from blueprint"} + with pytest.raises(BlueprintValidationError, match="no 'inventory' section"): + _resolve(topology_only, scope="inventory") + with pytest.raises(BlueprintValidationError, match="absent 'inventory' section"): + _resolve(topology_only, inventory_variant="secure") + assert _resolve(Blueprint.from_dict(_document()), scope="inventory").skipped == { + "topology": "not in scope 'inventory'" + } + + +def test_source_document_is_not_mutated() -> None: + document = _document() + original = copy.deepcopy(document) + blueprint = Blueprint.from_dict(document) + _resolve(blueprint, inventory_variant="secure", topology_variant="receiver") + _resolve(blueprint, inventory_variant="secure", topology_variant="receiver") + assert document == original + assert ( + blueprint.model_dump(exclude_unset=True, by_alias=True)["topology"]["receiver"] + == original["topology"]["receiver"] + ) + + +@pytest.mark.parametrize( + ("patch", "message"), + [ + ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"prot": 1}}}}, "Unknown custom setting"), + ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": "80"}}}}, "Expected an integer"), + ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": 0}}}}, "greater than or equal to 1"), + ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"driver_id": NMOS}}}}, "driver_id"), + ({"inventory": {"default": {"driver_id": "example.unknown-1.0"}}}, "Unknown driver"), + ({"inventory": {"default": {"custom_settings": {}}}}, "needs a driver_id"), + ({"inventory": {"default": {"driver_id": NMOS, "generic_settings": {"enable_https": "false"}}}}, "boolean"), + ( + {"inventory": {"default": {"driver_id": NMOS, "naming": {"endpoint_label": "x"}}}}, + "not valid in the 'inventory'", + ), + ], +) +def test_inventory_validation(patch: dict[str, Any], message: str) -> None: + with pytest.raises(BlueprintValidationError, match=message): + Blueprint.from_dict({"schema_version": 1, **patch}).validate_full() + + +@pytest.mark.parametrize( + ("document", "message"), + [ + ({"schema_version": 2, "topology": {"default": {}}}, "schema_version"), + ({"schema_version": 1}, "needs an 'inventory' or a 'topology'"), + ({"schema_version": 1, "topology": {"rx": {}}}, "must contain a 'default'"), + ({"schema_version": 1, "topology": {"default": {"unknown": 1}}}, "Unknown field"), + ({"schema_version": 1, "topology": {"default": {"device": {"label": None}}}}, "null is not supported"), + ( + {"schema_version": 1, "topology": {"default": {"device": {"tags": {"add": ["a"], "remove": ["a"]}}}}}, + "both add and remove", + ), + ({"schema_version": 1, "topology": {"default": {"vertices": [{"fields": {}}]}}}, "exactly one of"), + ( + {"schema_version": 1, "topology": {"default": {"naming": {"endpoint_label": "{device.nope}"}}}}, + "unknown field", + ), + ({"schema_version": 1, "topology": {"default": {"ip_vertex_mapping": {"a": []}}}}, "at least 1"), + ({"schema_version": "1", "topology": {"default": {}}}, "schema_version"), + ], +) +def test_document_validation(document: dict[str, Any], message: str) -> None: + with pytest.raises(BlueprintValidationError, match=message): + Blueprint.from_dict(document) + + +def test_full_validation_checks_unused_variants_and_reports_unavailable_processors() -> None: + document = _document() + document["topology"]["broken"] = {"vertex_processor": {"params": {"redundant_streams": "false"}}} + blueprint = Blueprint.from_dict(document) + with pytest.raises(BlueprintValidationError) as info: + blueprint.validate_full(ProcessorRegistry()) + codes = {issue.path: issue.code for issue in info.value.issues} + assert codes["topology.broken.vertex_processor.params.redundant_streams"] == "schema.bool_type" + assert codes["topology.custom.vertex_processor.processor_type"] == "processor.unavailable" + blueprint.model_copy() # structural load did not need the plugin + _resolve(blueprint, scope="inventory") # inventory-only use works without the plugin + + +def test_planning_requires_selected_processor() -> None: + blueprint = Blueprint.from_dict(_document()) + with pytest.raises(BlueprintValidationError, match="Unknown processor 'example-org.simple'"): + _resolve(blueprint, topology_variant="custom", registry=ProcessorRegistry()) + + +def test_schema_export_matches_runtime_and_package() -> None: + generated = Blueprint.json_schema(ProcessorRegistry()) + assert generated == published_json_schema() + packaged = Path(__file__).parents[2] / "src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json" + assert json.loads(packaged.read_text()) == generated + branches = generated["$defs"]["VertexProcessorSpec"]["allOf"] + assert [b["if"]["properties"]["processor_type"]["const"] for b in branches] == ["matrox.convertip.default"] + custom = Blueprint.json_schema(_registry()) + assert len(custom["$defs"]["VertexProcessorSpec"]["allOf"]) == 2 + assert "allOf" not in Blueprint.json_schema()["$defs"]["VertexProcessorSpec"] + + +def test_schema_can_restrict_processor_types_to_registered_ids() -> None: + open_spec = Blueprint.json_schema(_registry())["$defs"]["VertexProcessorSpec"] + assert "enum" not in open_spec["properties"]["processor_type"] + restricted = Blueprint.json_schema(_registry(), restrict_processor_types=True)["$defs"]["VertexProcessorSpec"] + processor_type = restricted["properties"]["processor_type"] + assert processor_type["enum"] == ["example-org.simple", "matrox.convertip.default"] + assert processor_type["type"] == "string" and "anyOf" not in processor_type + assert "processor_type" not in restricted.get("required", []) # variants may inherit it + assert len(restricted["allOf"]) == 2 + with pytest.raises(ValueError, match="requires a registry"): + Blueprint.json_schema(restrict_processor_types=True) + + +def test_documented_example_blueprint_is_valid() -> None: + example = Path(__file__).parents[2] / "docs/examples/07_blueprints/matrox-convertip.yml" + blueprint = Blueprint.load(example) + blueprint.validate_full(ProcessorRegistry()) + assert blueprint.variant_names("topology") == ["default", "receiver", "four-split"] From e26a62103badb5371fb8c367004b2ebec1127183 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:20:32 +0200 Subject: [PATCH 2/7] review --- docs/architecture/blueprints/concepts.md | 2 +- .../decisions/003-managed-fields.md | 4 + .../blueprints/decisions/005-app-gateways.md | 5 +- docs/architecture/blueprints/processors.md | 6 +- docs/architecture/blueprints/reference.md | 10 +- .../apps/inventory/app/app.py | 12 +- .../apps/inventory/errors.py | 22 +++ .../apps/inventory/inventory_api.py | 89 ++++++++----- .../blueprints/__init__.py | 2 + .../blueprints/engine.py | 51 ++++--- .../blueprints/errors.py | 9 ++ .../blueprints/inspect.py | 9 +- .../blueprints/inventory.py | 31 +++-- .../blueprints/models.py | 15 +-- .../blueprints/naming.py | 5 +- .../blueprints/processors/__init__.py | 3 +- .../blueprints/processors/matrox.py | 10 +- .../utils/cross_app_utils.py | 12 ++ tests/blueprints/conftest.py | 27 ++-- tests/blueprints/test_engine.py | 124 ++++++++++++++++- tests/blueprints/test_processors.py | 25 ++-- tests/inventory/__init__.py | 0 tests/inventory/test_inventory_api.py | 126 ++++++++++++++++++ 23 files changed, 477 insertions(+), 122 deletions(-) create mode 100644 src/videoipath_automation_tool/apps/inventory/errors.py create mode 100644 tests/inventory/__init__.py create mode 100644 tests/inventory/test_inventory_api.py diff --git a/docs/architecture/blueprints/concepts.md b/docs/architecture/blueprints/concepts.md index fcf5339..9c74233 100644 --- a/docs/architecture/blueprints/concepts.md +++ b/docs/architecture/blueprints/concepts.md @@ -163,7 +163,7 @@ replay in another process. [ADR-002](./decisions/002-plan-then-apply.md) fixes the phase order: 1. `inventory` — create or update the one record. A new id is stored on the result immediately. -2. `discovery` — poll until the scoped topology can be interpreted, bounded by `discovery_timeout` (default 30 seconds, poll every 1 second). +2. `discovery` — poll while the scoped topology is not ready yet (`TopologyNotReadyError`), bounded by `discovery_timeout` (default 30 seconds, poll every 1 second). Any other error fails at once. 3. `topology_sync` — add the device or synchronize it, according to `ApplyOptions.sync`. 4. `topology` — device and vertex edits in one `InspectTransaction`, then commit. 5. `module_tags` — `assignTag` / `unassignTag` actions, one tag at a time. These are not part of the topology commit. diff --git a/docs/architecture/blueprints/decisions/003-managed-fields.md b/docs/architecture/blueprints/decisions/003-managed-fields.md index f2acbf2..23f5252 100644 --- a/docs/architecture/blueprints/decisions/003-managed-fields.md +++ b/docs/architecture/blueprints/decisions/003-managed-fields.md @@ -49,3 +49,7 @@ not swallowed by Inventory's own authentication filter. value came from a previous blueprint". - Secrets are redacted in `FieldChange` and in `summary()`, and they are kept on the private captured work used to write. +- Secrets are write-only. The server masks them on read, so they are never + compared and never cause an update on their own. They are written on create + and with every update; `ApplyOptions(write_credentials=True)` forces a write + for a credential-only change. diff --git a/docs/architecture/blueprints/decisions/005-app-gateways.md b/docs/architecture/blueprints/decisions/005-app-gateways.md index bf3e1ac..c3227dc 100644 --- a/docs/architecture/blueprints/decisions/005-app-gateways.md +++ b/docs/architecture/blueprints/decisions/005-app-gateways.md @@ -23,8 +23,9 @@ treat a caller's unsaved edits as server truth, or flush them. record, resolves an SNMP configuration by exact label or id, creates with `add_device`, and updates with `update_device`. Label lookups use `find_device_id_by_label`. Address conflict checks use -`find_device_ids_by_addresses`, one read for every candidate spelling. An empty -inventory response is no match. +`find_device_ids_by_addresses`: one read for all addresses, compared after +normalization on both sides (IP literals in compressed form, other identifiers +case-insensitively). An empty inventory response is no match. `InspectGateway` is the only object that calls `app.inspect`. Scope data is built from fresh collector and lookup reads (`get_device_detail`, diff --git a/docs/architecture/blueprints/processors.md b/docs/architecture/blueprints/processors.md index e3ece3e..9160d7b 100644 --- a/docs/architecture/blueprints/processors.md +++ b/docs/architecture/blueprints/processors.md @@ -99,8 +99,10 @@ Precedence: explicit `topology.device` values override processor defaults, naming overrides proposed labels, and explicit `vertices` overrides win over both. -Raise `ProcessorInputError` when the layout is unsupported or still incomplete. -While discovery is deferred, the engine retries until `discovery_timeout`. +Raise `TopologyNotReadyError` when discovery is still incomplete (for example, +vertices without an edit form, or no codec vertices yet). While topology work is +deferred, the engine retries only this error, until `discovery_timeout`. Raise +`ProcessorInputError` when the layout is unsupported; the apply fails at once. ## 4. Registration diff --git a/docs/architecture/blueprints/reference.md b/docs/architecture/blueprints/reference.md index c0e70ae..9e264fb 100644 --- a/docs/architecture/blueprints/reference.md +++ b/docs/architecture/blueprints/reference.md @@ -82,7 +82,7 @@ your source system knows. | `label`, `description` | Instance facts used by naming. | | `inventory_id` | The Inventory record that already belongs to this device. If that record is missing, planning fails. The engine will not create a second one in its place. Leave the field out when a selected inventory section should create the record. | | `management_address`, `alternative_addresses` | Connection addresses. The driver decides the string format, so these are not always IP addresses. An alternative address is a string or an `AlternativeAddress(address, credentials)`. Leave the field out to leave the server value alone. Pass `[]` to clear alternative addresses. A supplied list is the full desired set and also replaces per-address credentials: a plain string carries none, so an existing credential is cleared unless that address is an `AlternativeAddress`. An address listed twice fails. | -| `credentials` | Runtime `Credentials(username, password)`. Credentials stay out of YAML, and the engine redacts them in plans, results, and errors. | +| `credentials` | Runtime `Credentials(username, password)`. Credentials stay out of YAML, and the engine redacts them in plans, results, and errors. Secrets are write-only: the server masks them on read, so a plan never compares them. They are written on create and with every update; a credential change alone needs `ApplyOptions(write_credentials=True)`. | | `topology` | `DeviceTarget(device_id)` or `ModuleTarget(device_id, module_id)`, using exact Inspect ids. Leave it out to target the Inventory id, including an id this same plan creates. | | `module_position` | Module context you supply for naming and mappings. It is a string, such as `"0"`, `"A1"`, or `"1/2"`. | | `inventory_overrides` | Per-device Inventory settings that are not secrets (`InventorySettings`): `custom_settings`, `generic_settings`, `snmp`, `metadata`, `active`. Applied after the selected variant. | @@ -389,6 +389,7 @@ plan = engine.plan( | `discovery_timeout` | `30.0` | Seconds to wait for discovery during apply. | | `poll_interval` | `1.0` | Seconds between discovery polls. | | `naming_collisions` | `"reject"` | `"allow"` lets duplicate endpoint labels through. | +| `write_credentials` | `False` | Write the supplied credentials to an existing record even when nothing else changes (password rotation). | Planning reads the current state. It does not write, synchronize, stage snapshot edits, or create catalog entries. The plan lists every phase with the exact @@ -400,7 +401,7 @@ Apply then runs the phases in this order ([ADR-002](./decisions/002-plan-then-apply.md)): 1. `inventory` creates or updates the record. A new id is stored on the result immediately. -2. `discovery` waits, bounded by `discovery_timeout`. +2. `discovery` waits while the topology is not ready yet (`TopologyNotReadyError`), bounded by `discovery_timeout`. Other errors fail at once. 3. `topology_sync` adds the device to the topology, or synchronizes it. 4. `topology` edits the device and its vertices in one Inspect transaction, with conflict checking. 5. `module_tags` assigns or unassigns tags as separate actions. @@ -494,7 +495,8 @@ All errors derive from `BlueprintError`: |---|---| | `BlueprintValidationError` | A document, naming expression, or parameter is invalid. `.issues` lists `ValidationIssue`s; `.codes` lists their codes. | | `BlueprintTargetError` | A binding is missing, ambiguous, or outside the target scope. | -| `ProcessorInputError` | The discovered topology is unsupported or still incomplete. | +| `ProcessorInputError` | The discovered topology is unsupported. | +| `TopologyNotReadyError` | The discovered topology is still incomplete. Retried while topology work is deferred. | | `BlueprintCapabilityError` | The engine cannot perform the requested operation. | | `BlueprintConflictError` | The plan is stale: the server changed since planning. | | `BlueprintApplyError` | An apply failed. `.result` is the `ApplyResult`. | @@ -580,4 +582,4 @@ Everything below is importable from `videoipath_automation_tool.blueprints`. | Options and results | `ApplyOptions`, `ApplyResult`, `PhaseResult`, `PlannedPhase`, `PlannedOperation`, `FieldChange`, `InterfaceBinding`, `Diagnostic` | | Naming | `NamingScheme`, `DEFAULT_NAMING`, `Text`, `Field`, `Join`, `NameContext`, `NameRenderer` | | Processors | `VertexProcessor`, `ProcessorRegistry`, `ProcessingContext`, `ProcessorResult`, `VertexEdit`, `EndpointIdentity`, `SourceFacts`, `DriverContext`, `DeviceRecord`, `ModuleRecord`, `PortRecord`, `VertexRecord`, `MatroxConvertIPProcessor`, `MatroxConvertIPParams` | -| Errors | `BlueprintError`, `BlueprintValidationError`, `ValidationIssue`, `BlueprintTargetError`, `ProcessorInputError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | +| Errors | `BlueprintError`, `BlueprintValidationError`, `ValidationIssue`, `BlueprintTargetError`, `ProcessorInputError`, `TopologyNotReadyError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | diff --git a/src/videoipath_automation_tool/apps/inventory/app/app.py b/src/videoipath_automation_tool/apps/inventory/app/app.py index 9676529..ee0cfd8 100644 --- a/src/videoipath_automation_tool/apps/inventory/app/app.py +++ b/src/videoipath_automation_tool/apps/inventory/app/app.py @@ -9,6 +9,7 @@ InventoryCreateDeviceFromDiscoveredDeviceMixin, ) from videoipath_automation_tool.apps.inventory.app.get_device import InventoryGetDeviceMixin +from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError from videoipath_automation_tool.apps.inventory.inventory_api import InventoryAPI from videoipath_automation_tool.apps.inventory.model.drivers import CustomSettings, CustomSettingsType, DriverLiteral from videoipath_automation_tool.apps.inventory.model.global_snmp_config import SnmpConfiguration @@ -127,7 +128,9 @@ def update_device( try: existing_device = self._inventory_api.get_device(device_id=device.device_id, config_only=True) except ValueError as e: - raise ValueError(f"Failed to retrieve existing device configuration from Inventory: {e}") from None + raise InventoryWriteNotAppliedError( + f"Failed to retrieve existing device configuration from Inventory: {e}", operation="update" + ) from None if compare_config: comparison = self.diff_device_configuration(reference_device=existing_device, staged_device=device) @@ -232,10 +235,11 @@ def find_device_id_by_label( else: raise ValueError(f"Invalid label_search_mode: {label_search_mode}") - def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> list[str]: - """Device ids whose management or alternate address matches any of ``addresses``. + def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> dict[str, list[str]]: + """Map each of ``addresses`` to the sorted ids of devices whose management or alternate + address matches it (IP literals normalized, other identifiers case-insensitive). - One server read for every candidate spelling. An empty inventory response is no match. + One server read for all addresses. An empty inventory response is no match. """ return self._inventory_api.find_device_ids_by_addresses(addresses) diff --git a/src/videoipath_automation_tool/apps/inventory/errors.py b/src/videoipath_automation_tool/apps/inventory/errors.py new file mode 100644 index 0000000..68bfbd6 --- /dev/null +++ b/src/videoipath_automation_tool/apps/inventory/errors.py @@ -0,0 +1,22 @@ +"""Typed Inventory errors.""" + +from __future__ import annotations + +from typing import Literal + +InventoryWriteOperation = Literal["add", "update"] + + +class InventoryWriteNotAppliedError(ValueError): + """An Inventory write was rejected by the server or not attempted, so it did not change the record. + + ``operation`` names the write that was not applied. Subclasses ``ValueError`` so existing callers + that catch ``ValueError`` keep working. + """ + + def __init__(self, message: str, *, operation: InventoryWriteOperation) -> None: + self.operation: InventoryWriteOperation = operation + super().__init__(message) + + +__all__ = ["InventoryWriteNotAppliedError", "InventoryWriteOperation"] diff --git a/src/videoipath_automation_tool/apps/inventory/inventory_api.py b/src/videoipath_automation_tool/apps/inventory/inventory_api.py index 60ceecd..b09a37d 100644 --- a/src/videoipath_automation_tool/apps/inventory/inventory_api.py +++ b/src/videoipath_automation_tool/apps/inventory/inventory_api.py @@ -8,6 +8,7 @@ from pydantic import IPvAnyAddress from typing_extensions import deprecated +from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError from videoipath_automation_tool.apps.inventory.inventory_utils import ( construct_driver_id_from_info, extract_driver_info_from_id, @@ -21,7 +22,11 @@ from videoipath_automation_tool.apps.inventory.model.inventory_request_rpc import InventoryRequestRpc from videoipath_automation_tool.connector.models.response_rpc import ResponseRPC from videoipath_automation_tool.connector.vip_connector import VideoIPathConnector -from videoipath_automation_tool.utils.cross_app_utils import create_fallback_logger, extract_natural_sort_key +from videoipath_automation_tool.utils.cross_app_utils import ( + create_fallback_logger, + extract_natural_sort_key, + normalize_address, +) from videoipath_automation_tool.validators.device_id import validate_device_id @@ -167,7 +172,9 @@ def add_device( response = self.vip_connector.rpc.post("/api/updateDevices", body=body) if response.header.status != "OK": - raise ValueError(f"Failed to add device to VideoIPath-Inventory. Error: {response}") + raise InventoryWriteNotAppliedError( + f"Failed to add device to VideoIPath-Inventory. Error: {response}", operation="add" + ) online_device = self._fetch_device_config_by_uuid(uuid=tracking_id) @@ -251,7 +258,9 @@ def update_device( response = self.vip_connector.rpc.post("/api/updateDevices", body=body) if response.header.status != "OK": - raise ValueError(f"Failed to update device in VideoIPath-Inventory. Error: {response}") + raise InventoryWriteNotAppliedError( + f"Failed to update device in VideoIPath-Inventory. Error: {response}", operation="update" + ) online_device = self.get_device( device_id=device.configuration.id, @@ -548,26 +557,28 @@ def get_device_id_by_address(self, address: str, include_alt_addresses: bool = T Optional[str | List[str]]: Device id, None if address does not exist, List of device ids if multiple devices with the same address exist """ if include_alt_addresses: - url = "/rest/v2/data/config/devman/devices/*/config/cinfo/address,altAddresses,altAddresses/**" - # altAddresses contains all addresses from altAddressesWithAuth, therefore no need to fetch altAddressesWithAuth + index = self._read_device_addresses() + if index is None: + raise ValueError("Response data is empty.") + device_ids = [device_id for device_id, known in index if address in known] else: escaped_address = urllib.parse.quote(address, safe="") url = f"/rest/v2/data/config/devman/devices/* where config.cinfo.address='{escaped_address}' /_id" - response = self.vip_connector.rest.get(url) + response = self.vip_connector.rest.get(url) - if response.data and isinstance(response.data["config"]["devman"]["devices"]["_items"], list): - devices = response.data["config"]["devman"]["devices"]["_items"] - else: - raise ValueError("Response data is empty.") + if response.data and isinstance(response.data["config"]["devman"]["devices"]["_items"], list): + devices = response.data["config"]["devman"]["devices"]["_items"] + else: + raise ValueError("Response data is empty.") - device_ids = [] - for device in devices: - if ( - address in device["config"]["cinfo"].get("altAddresses") - or address == device["config"]["cinfo"]["address"] - ): - device_ids.append(device["_id"]) + device_ids = [] + for device in devices: + if ( + address in device["config"]["cinfo"].get("altAddresses") + or address == device["config"]["cinfo"]["address"] + ): + device_ids.append(device["_id"]) if len(device_ids) == 0: return None @@ -576,15 +587,30 @@ def get_device_id_by_address(self, address: str, include_alt_addresses: bool = T else: return device_ids - def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> list[str]: - """Device ids whose management or alternate address equals any of ``addresses``. + def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> dict[str, list[str]]: + """Map each of ``addresses`` to the sorted ids of devices whose management or alternate + address matches it. - One bulk read. A missing item list is no match. A missing ``altAddresses`` list on a device - is treated as empty. + Addresses are compared after :func:`normalize_address` on both sides (IP literals in compressed + form, other identifiers case-insensitively). One bulk read for all addresses. Empty addresses + are ignored; a missing item list is no match. """ - wanted = {address for address in addresses if address} + wanted = {address: normalize_address(address) for address in addresses if address} if not wanted: - return [] + return {} + normalized_index = [ + (device_id, {normalize_address(known) for known in known_addresses}) + for device_id, known_addresses in self._read_device_addresses() or [] + ] + return { + address: sorted(device_id for device_id, known in normalized_index if normalized in known) + for address, normalized in wanted.items() + } + + def _read_device_addresses(self) -> list[tuple[str, list[str]]] | None: + """``(device_id, [address, *altAddresses])`` for every Inventory device in one bulk read, or + ``None`` when the response carries no item list. A missing ``altAddresses`` list is empty.""" + # altAddresses contains all addresses from altAddressesWithAuth, therefore no need to fetch altAddressesWithAuth url = "/rest/v2/data/config/devman/devices/*/config/cinfo/address,altAddresses,altAddresses/**" response = self.vip_connector.rest.get(url) devices = ( @@ -593,17 +619,18 @@ def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> list[str]: else None ) if not isinstance(devices, list): - return [] - found: set[str] = set() + return None + index: list[tuple[str, list[str]]] = [] for device in devices: + if not isinstance(device, dict) or not isinstance(device.get("_id"), str): + continue cinfo = device.get("config", {}).get("cinfo", {}) - known = {cinfo.get("address")} - alternates = cinfo.get("altAddresses") or [] + known = [cinfo["address"]] if isinstance(cinfo.get("address"), str) else [] + alternates = cinfo.get("altAddresses") if isinstance(alternates, list): - known.update(alternates) - if known & wanted and isinstance(device.get("_id"), str): - found.add(device["_id"]) - return sorted(found) + known.extend(address for address in alternates if isinstance(address, str)) + index.append((device["_id"], known)) + return index def get_device_id_by_meta_field_value(self, meta_field: str, value: str) -> Optional[str | List[str]]: """Method to get a device id by given meta field value from VideoIPath-Inventory diff --git a/src/videoipath_automation_tool/blueprints/__init__.py b/src/videoipath_automation_tool/blueprints/__init__.py index c69f71c..ed337cc 100644 --- a/src/videoipath_automation_tool/blueprints/__init__.py +++ b/src/videoipath_automation_tool/blueprints/__init__.py @@ -19,6 +19,7 @@ BlueprintTargetError, BlueprintValidationError, ProcessorInputError, + TopologyNotReadyError, ValidationIssue, ) from videoipath_automation_tool.blueprints.models import ( @@ -119,6 +120,7 @@ "SourceFacts", "TagDelta", "Text", + "TopologyNotReadyError", "ValidationIssue", "VertexEdit", "VertexPatch", diff --git a/src/videoipath_automation_tool/blueprints/engine.py b/src/videoipath_automation_tool/blueprints/engine.py index 3a2e617..f8c0352 100644 --- a/src/videoipath_automation_tool/blueprints/engine.py +++ b/src/videoipath_automation_tool/blueprints/engine.py @@ -25,6 +25,7 @@ from pydantic import BaseModel, ConfigDict, PrivateAttr from videoipath_automation_tool.apps.inspect.errors import InspectCommitConflictError, InspectCommitError +from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice from videoipath_automation_tool.blueprints.errors import ( BlueprintApplyError, @@ -33,7 +34,7 @@ BlueprintError, BlueprintTargetError, BlueprintValidationError, - ProcessorInputError, + TopologyNotReadyError, ) from videoipath_automation_tool.blueprints.inspect import ( InspectGateway, @@ -86,10 +87,12 @@ class BlueprintApp(Protocol): """The app interface the engine needs; :class:`VideoIPathApp` satisfies it unchanged.""" @property - def inventory(self) -> Any: ... + def inventory(self) -> Any: + """The Inventory app (``InventoryApp``).""" @property - def inspect(self) -> Any: ... + def inspect(self) -> Any: + """The Inspect app (``InspectApp``).""" class BlueprintPlan(BaseModel): @@ -443,7 +446,12 @@ def _plan_inventory(self, phases: list[PlannedPhase]) -> tuple[InventoryWork | N else None ) work, current = plan_inventory( - self._inventory(), resolved.inventory, self._captured.device, label=label, description=description + self._inventory(), + resolved.inventory, + self._captured.device, + label=label, + description=description, + write_credentials=self._captured.options.write_credentials, ) operation = work.operation phases.append( @@ -726,7 +734,7 @@ def _run_topology(self) -> None: phase.status = "no_change" self._run_module_tags(gateway, work) - if self._wrote and ("topology" in self._wrote or "module_tags" in self._wrote): + if "topology" in self._wrote or "module_tags" in self._wrote: try: self._mismatches.extend(verify_topology(gateway.read_scope(target), work)) except Exception as exc: # noqa: BLE001 - verification never turns an applied write into a failure @@ -779,13 +787,15 @@ def _discover_and_materialize(self, gateway: InspectGateway, target: TopologyTar ) discovery.status = "completed" return work - except (ProcessorInputError, BlueprintTargetError) as exc: - if self._unknown or isinstance(exc, _NoRetry): + except TopologyNotReadyError as exc: + if self._unknown: raise remaining = deadline - self._executor._clock() if remaining <= 0: self._current = "discovery" - raise type(exc)(f"Topology not ready after {options.discovery_timeout:g}s: {exc}") from exc + raise TopologyNotReadyError( + f"Topology not ready after {options.discovery_timeout:g}s: {exc}" + ) from exc self._executor._sleep(min(options.poll_interval, remaining)) def _ensure_membership(self, gateway: InspectGateway, target: TopologyTarget) -> None: @@ -794,13 +804,13 @@ def _ensure_membership(self, gateway: InspectGateway, target: TopologyTarget) -> if isinstance(target, ModuleTarget): # Never add or synchronize a whole parent device to reach a module target. if not gateway.in_topology(target.device_id): - raise _NoRetry( + raise BlueprintTargetError( f"Parent device '{target.device_id}' of module '{target.module_id}' is not in the topology." ) return if not gateway.in_topology(target.device_id): if sync == "none": - raise _NoRetry(f"Inspect device '{target.device_id}' is not in the topology (sync='none').") + raise BlueprintTargetError(f"Inspect device '{target.device_id}' is not in the topology (sync='none').") with self._write(): gateway.add_to_topology(target.device_id) self._record_sync(phase, "add_to_topology", target.device_id) @@ -900,27 +910,14 @@ def _inventory(self) -> InventoryGateway: _PHASE_ORDER: tuple[str, ...] = ("inventory", "discovery", "topology_sync", "topology", "module_tags", "verification") -class _NoRetry(BlueprintTargetError): - """A target error that waiting cannot resolve.""" - - def _known_rejection(exc: Exception, *, create: bool) -> bool: """Whether a write error proves the server did not apply the change.""" if isinstance(exc, (InspectCommitError, InspectCommitConflictError, BlueprintError)): return True - if isinstance(exc, ValueError): - message = str(exc) - return ( - message.startswith("Failed to add device") - if create - else message.startswith( - ( - "Failed to update device", - "Failed to add device", - "Failed to retrieve existing device configuration", - ) - ) - ) + if isinstance(exc, InventoryWriteNotAppliedError): + # A create is only known to be rejected when the add itself failed: the follow-up update + # (tracking-id cleanup) runs after the record already exists. + return exc.operation == "add" if create else True return False diff --git a/src/videoipath_automation_tool/blueprints/errors.py b/src/videoipath_automation_tool/blueprints/errors.py index 6cdaac3..0fcdad7 100644 --- a/src/videoipath_automation_tool/blueprints/errors.py +++ b/src/videoipath_automation_tool/blueprints/errors.py @@ -70,6 +70,14 @@ class ProcessorInputError(BlueprintError): """A processor cannot interpret the observed topology (unsupported or incomplete layout).""" +class TopologyNotReadyError(BlueprintError): + """The discovered topology is still incomplete; waiting for discovery may resolve it. + + While topology work is deferred, ``apply()`` retries this error until ``discovery_timeout``. + Every other error fails the attempt immediately. + """ + + class BlueprintCapabilityError(BlueprintError): """A requested operation is not supported by the adapter or the server.""" @@ -99,5 +107,6 @@ def __init__(self, result: ApplyResult, message: str) -> None: "BlueprintTargetError", "BlueprintValidationError", "ProcessorInputError", + "TopologyNotReadyError", "ValidationIssue", ] diff --git a/src/videoipath_automation_tool/blueprints/inspect.py b/src/videoipath_automation_tool/blueprints/inspect.py index 02784a4..ab628dc 100644 --- a/src/videoipath_automation_tool/blueprints/inspect.py +++ b/src/videoipath_automation_tool/blueprints/inspect.py @@ -31,6 +31,7 @@ BlueprintTargetError, BlueprintValidationError, ProcessorInputError, + TopologyNotReadyError, ValidationIssue, ) from videoipath_automation_tool.blueprints.models import ( @@ -243,11 +244,11 @@ def staged_edit_keys(self) -> set[tuple[str, str]]: def add_to_topology(self, device_id: str) -> None: if not self._app.add_devices_to_topology([device_id], sync=False): - raise BlueprintTargetError(f"addDevices reported failure for '{device_id}' (not discovered yet?).") + raise TopologyNotReadyError(f"addDevices reported failure for '{device_id}' (not discovered yet?).") def sync(self, device_id: str, *, add_only: bool) -> None: if not self._app.sync_devices([device_id], add_only=add_only): - raise BlueprintTargetError(f"syncDevices reported failure for '{device_id}'.") + raise TopologyNotReadyError(f"syncDevices reported failure for '{device_id}'.") def commit(self, work: TopologyWork) -> Any: with self._app.transaction() as tx: @@ -325,7 +326,9 @@ def resolve_interfaces( ) break else: - raise BlueprintTargetError( + # No ports at all means discovery has not produced them yet; a miss among ports is final. + error = TopologyNotReadyError if not scope.ports else BlueprintTargetError + raise error( f"Interface '{key}': no IP port in {_scope_label(scope)} matches {', '.join(map(repr, attempted))}." ) return bindings diff --git a/src/videoipath_automation_tool/blueprints/inventory.py b/src/videoipath_automation_tool/blueprints/inventory.py index 3162f20..eea078d 100644 --- a/src/videoipath_automation_tool/blueprints/inventory.py +++ b/src/videoipath_automation_tool/blueprints/inventory.py @@ -7,6 +7,10 @@ performs). The generic comparison of ``update_device`` is bypassed once a managed change is known, so explicit credential changes cannot be swallowed by its authentication filtering. +Secrets (passwords, alternative-address credentials) are write-only: the server masks them on read, so +they are never compared. They are written on create and with every update, and they cause an update on +their own only with ``ApplyOptions(write_credentials=True)``. + Conflict detection is client-side (re-read and compare before writing); a read-to-write race remains. """ @@ -35,7 +39,6 @@ CatalogId, FieldChange, PlannedOperation, - normalize_address, ) from videoipath_automation_tool.blueprints.resolution import ResolvedInventory @@ -103,9 +106,11 @@ def ids_with_label(self, label: str) -> list[str]: found = self._app.find_device_id_by_label(label, label_search_mode="user_defined_label_only") return _as_list(found) - def ids_with_address(self, address: str) -> list[str]: - candidates = {address.strip(), normalize_address(address)} - return self._app.find_device_ids_by_addresses(candidates) + def ids_by_addresses(self, addresses: list[str]) -> dict[str, list[str]]: + """Device ids per address (normalized comparison), in one read.""" + if not addresses: + return {} + return self._app.find_device_ids_by_addresses(addresses) def resolve_snmp(self, reference: str | CatalogId) -> str: """Exact SNMP configuration id for a label or ``{id: ...}``; missing/ambiguous fail.""" @@ -144,8 +149,13 @@ def plan_inventory( *, label: str | None, description: str | None, + write_credentials: bool = False, ) -> tuple[InventoryWork, InventoryDevice | None]: - """Compute the Inventory work (read-only). Returns the work and the current record (if bound).""" + """Compute the Inventory work (read-only). Returns the work and the current record (if bound). + + Secrets are not compared (see the module docstring); for an existing record they are listed as + changes only when ``write_credentials`` is set. + """ desired = _desired_values(gateway, resolved, device, label=label, description=description) current: InventoryDevice | None = None @@ -174,7 +184,8 @@ def plan_inventory( changes = [ _field_change(key, get_field(current, key) if current is not None else None, item) for key, item in desired.items() - if current is None or not values_equal(get_field(current, key), item.value) + if current is None + or (write_credentials if is_sensitive(key) else not values_equal(get_field(current, key), item.value)) ] _check_conflicts(gateway, device, desired, changes, own_id=device.inventory_id) @@ -217,7 +228,7 @@ def check_baseline(work: InventoryWork, fresh: InventoryDevice) -> None: def verify(device: InventoryDevice, desired: dict[str, DesiredValue]) -> list[str]: - """Managed fields whose read-back differs (secrets are not compared: the server may mask them).""" + """Managed fields whose read-back differs (secrets are not compared: the server masks them).""" return sorted( key for key, item in desired.items() @@ -332,8 +343,9 @@ def _check_conflicts( addresses.append(desired["address"].value) if "alt_addresses" in changed: addresses.extend(desired["alt_addresses"].value) + found = gateway.ids_by_addresses(addresses) for address in addresses: - others = [i for i in gateway.ids_with_address(address) if i != own_id] + others = [i for i in found.get(address, []) if i != own_id] if others: conflicts.append(f"address '{address}' is used by {', '.join(others)}") if conflicts: @@ -342,7 +354,8 @@ def _check_conflicts( def _baseline(device: InventoryDevice, desired: dict[str, DesiredValue]) -> dict[str, str]: - baseline = {key: _digest(get_field(device, key)) for key in desired} + """Fingerprints of the managed non-secret values (masked secrets carry no information) and driver.""" + baseline = {key: _digest(get_field(device, key)) for key in desired if not is_sensitive(key)} baseline["driver_id"] = _digest(device.driver_id) return baseline diff --git a/src/videoipath_automation_tool/blueprints/models.py b/src/videoipath_automation_tool/blueprints/models.py index 95cc04e..dd6e2a4 100644 --- a/src/videoipath_automation_tool/blueprints/models.py +++ b/src/videoipath_automation_tool/blueprints/models.py @@ -10,7 +10,6 @@ from __future__ import annotations -import ipaddress from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Literal @@ -41,6 +40,7 @@ TOPOLOGY_NAMING_ENTRIES, BlueprintNaming, ) +from videoipath_automation_tool.utils.cross_app_utils import normalize_address from videoipath_automation_tool.validators.device_id import validate_device_id if TYPE_CHECKING: @@ -459,12 +459,15 @@ class ApplyOptions(_StrictModel): elements; ``"reconcile"`` permits a full sync. Service conflicts are always handled strictly. ``discovery_timeout`` / ``poll_interval`` bound the wait for driver discovery (seconds). ``naming_collisions="allow"`` permits duplicate endpoint labels within a device. + ``write_credentials=True`` writes the supplied credentials to an existing Inventory record even when + nothing else changes (secrets are masked on read, so a plan cannot tell whether they differ). """ sync: SyncPolicy = "add_only" discovery_timeout: Annotated[float, Field(gt=0)] = 30.0 poll_interval: Annotated[float, Field(gt=0)] = 1.0 naming_collisions: Literal["reject", "allow"] = "reject" + write_credentials: bool = False # --- Plan and result records --- @@ -554,15 +557,6 @@ def phase(self, name: PhaseName) -> PhaseResult | None: # --- Helpers --- -def normalize_address(address: str) -> str: - """Normalize IP literals (no DNS); other identifiers compare case-insensitively.""" - candidate = address.strip() - try: - return ipaddress.ip_address(candidate.strip("[]")).compressed - except ValueError: - return candidate.casefold() - - def issues_from_pydantic( exc: ValidationError, *, @@ -675,6 +669,7 @@ def _friendly_message(error: Any) -> str: "PlannedOperation", "PlannedPhase", "ProcessorResult", + "Scope", "SnmpSelection", "TagDelta", "TagSpec", diff --git a/src/videoipath_automation_tool/blueprints/naming.py b/src/videoipath_automation_tool/blueprints/naming.py index 6478e34..d9d3671 100644 --- a/src/videoipath_automation_tool/blueprints/naming.py +++ b/src/videoipath_automation_tool/blueprints/naming.py @@ -116,7 +116,8 @@ def value(self, path: str) -> Any: class NameRenderer(Protocol): """Trusted Python escape hatch for one naming entry; never loaded from YAML.""" - def render(self, context: NameContext) -> str: ... + def render(self, context: NameContext) -> str: + """Return the name for ``context``.""" NamingEntry = NameExpr | InstanceOf[NameRenderer] @@ -412,7 +413,9 @@ def _format_value(field: Field, value: Any) -> str: __all__ = [ "DEFAULT_NAMING", + "INVENTORY_NAMING_ENTRIES", "NAMING_ENTRIES", + "TOPOLOGY_NAMING_ENTRIES", "BlueprintNaming", "Field", "Join", diff --git a/src/videoipath_automation_tool/blueprints/processors/__init__.py b/src/videoipath_automation_tool/blueprints/processors/__init__.py index fe2092b..8b9f3ad 100644 --- a/src/videoipath_automation_tool/blueprints/processors/__init__.py +++ b/src/videoipath_automation_tool/blueprints/processors/__init__.py @@ -203,7 +203,8 @@ class VertexProcessor(ABC, Generic[ParamsT]): @abstractmethod def process(self, context: ProcessingContext, params: ParamsT) -> ProcessorResult: """Interpret the scoped topology and propose configuration. Raise - :class:`ProcessorInputError` when the observed topology is unsupported or incomplete.""" + :class:`TopologyNotReadyError` when discovery is still incomplete (retried while topology work is + deferred) and :class:`ProcessorInputError` when the layout is unsupported (fails immediately).""" class ProcessorRegistry: diff --git a/src/videoipath_automation_tool/blueprints/processors/matrox.py b/src/videoipath_automation_tool/blueprints/processors/matrox.py index b6d60a9..fab1354 100644 --- a/src/videoipath_automation_tool/blueprints/processors/matrox.py +++ b/src/videoipath_automation_tool/blueprints/processors/matrox.py @@ -27,7 +27,11 @@ from pydantic import BaseModel, ConfigDict -from videoipath_automation_tool.blueprints.errors import BlueprintCapabilityError, ProcessorInputError +from videoipath_automation_tool.blueprints.errors import ( + BlueprintCapabilityError, + ProcessorInputError, + TopologyNotReadyError, +) from videoipath_automation_tool.blueprints.models import ( DevicePatch, Diagnostic, @@ -72,13 +76,13 @@ def process(self, context: ProcessingContext, params: MatroxConvertIPParams) -> unknown = [vertex.id for vertex in context.vertices if vertex.kind is None] if unknown: - raise ProcessorInputError( + raise TopologyNotReadyError( f"Matrox ConvertIP: unknown kind for vertices {', '.join(sorted(unknown))} in {context.scope_label} " "(no edit form; incomplete discovery?)." ) codecs = context.find_vertices(kind="codec") if not codecs: - raise ProcessorInputError( + raise TopologyNotReadyError( f"Matrox ConvertIP: no codec vertices in {context.scope_label} (incomplete discovery or unsupported layout)." ) media = self._classify_media(codecs, params, diagnostics) diff --git a/src/videoipath_automation_tool/utils/cross_app_utils.py b/src/videoipath_automation_tool/utils/cross_app_utils.py index 7d2cbc4..dcfa633 100644 --- a/src/videoipath_automation_tool/utils/cross_app_utils.py +++ b/src/videoipath_automation_tool/utils/cross_app_utils.py @@ -1,3 +1,4 @@ +import ipaddress import logging import re import uuid @@ -20,6 +21,17 @@ def generate_uuid_4(): return str(uuid.uuid4()) +# --- Address comparison --- +def normalize_address(address: str) -> str: + """Normalize a device address for comparison: IP literals in compressed form (no DNS lookup), + any other identifier case-insensitively.""" + candidate = address.strip() + try: + return ipaddress.ip_address(candidate.strip("[]")).compressed + except ValueError: + return candidate.casefold() + + # --- Natural Sort Device ID list --- def extract_natural_sort_key(s: str): return [int(text) if text.isdigit() else text.lower() for text in re.split(r"(\d+)", s)] diff --git a/tests/blueprints/conftest.py b/tests/blueprints/conftest.py index e1ea743..546fa19 100644 --- a/tests/blueprints/conftest.py +++ b/tests/blueprints/conftest.py @@ -27,6 +27,7 @@ from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyResponse from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice from videoipath_automation_tool.blueprints import BlueprintEngine +from videoipath_automation_tool.utils.cross_app_utils import normalize_address NMOS = "com.nevion.NMOS_multidevice-0.1.0" HEADER = {"auth": True, "caption": "OK", "code": "OK", "id": "0", "ok": True, "user": "test-user"} @@ -89,18 +90,24 @@ def get_global_snmp_config_id_by_label(self, label: str) -> Any: def get_global_snmp_config_label_by_id(self, snmp_config_id: str) -> str | None: return self.snmp.get(snmp_config_id) - def find_device_ids_by_addresses(self, addresses: Any) -> list[str]: + def find_device_ids_by_addresses(self, addresses: Any) -> dict[str, list[str]]: self.address_lookups += 1 - ids = [ - i - for i, device in self.devices.items() - if any( - address == device.configuration.config.cinfo.address - or address in device.configuration.config.cinfo.altAddresses - for address in addresses + result: dict[str, list[str]] = {} + for address in addresses: + wanted = normalize_address(address) + result[address] = sorted( + i + for i, device in self.devices.items() + if any( + normalize_address(known) == wanted + for known in [ + device.configuration.config.cinfo.address, + *device.configuration.config.cinfo.altAddresses, + ] + if known + ) ) - ] - return sorted(set(ids)) + return result def _maybe_fail(self) -> None: if self.fail_next_write is not None: diff --git a/tests/blueprints/test_engine.py b/tests/blueprints/test_engine.py index daf4e60..b7ec0aa 100644 --- a/tests/blueprints/test_engine.py +++ b/tests/blueprints/test_engine.py @@ -13,6 +13,7 @@ from tests.blueprints.conftest import NMOS, FakeInspectServer, FakeInventory, make_inspect_app, matrox_layout from videoipath_automation_tool.apps.inspect.snapshot import InspectSnapshot +from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError from videoipath_automation_tool.blueprints import ( AlternativeAddress, ApplyOptions, @@ -27,6 +28,7 @@ BlueprintTargetError, BlueprintValidationError, Credentials, + DeviceRecord, DeviceTarget, EndpointIdentity, ModuleTarget, @@ -34,11 +36,13 @@ ProcessingContext, ProcessorInputError, ProcessorResult, + TopologyNotReadyError, VertexEdit, VertexPatch, VertexProcessor, published_json_schema, ) +from videoipath_automation_tool.blueprints.inspect import ScopeData, resolve_interfaces MATROX = { "schema_version": 1, @@ -223,9 +227,17 @@ def test_update_preserves_unmanaged_fields_and_redacts_secrets( ], ) plan = engine.plan(device, Blueprint.from_dict(document), scope="inventory") - summary = plan.summary() - assert "test-password" not in summary and "alt-password" not in summary and "********" in summary - assert "test-password" not in repr(device) and "test-password" not in plan.model_dump_json() + fields = {change.field for change in plan.phase("inventory").operations[0].changes} + assert "credentials.password" not in fields and "alt_addresses_with_auth" not in fields + shown = engine.plan( + device, Blueprint.from_dict(document), scope="inventory", options=ApplyOptions(write_credentials=True) + ) + for reviewed in (plan, shown): + summary = reviewed.summary() + assert "test-password" not in summary and "alt-password" not in summary + assert "test-password" not in reviewed.model_dump_json() + assert "********" in shown.summary() + assert "test-password" not in repr(device) plan.apply() stored = inventory.devices["device1"].configuration assert stored.config.customSettings.port == 8080 and stored.config.customSettings.indices_in_ids is False @@ -235,15 +247,52 @@ def test_update_preserves_unmanaged_fields_and_redacts_secrets( assert stored.config.cinfo.altAddressesWithAuth == [ {"address": "192.0.2.21", "authentication": {"user": "test-user", "password": "alt-password"}} ] - # A changed password alone still writes; omission preserves it. + # Secrets are write-only: a changed password alone writes only with write_credentials; omission preserves it. changed = device.model_copy(update={"credentials": Credentials(username="test-user", password="new-password")}) - assert engine.plan(changed, Blueprint.from_dict(document), scope="inventory").phase("inventory").status == "planned" + assert engine.plan(changed, Blueprint.from_dict(document), scope="inventory").phase("inventory").status == ( + "no_change" + ) + rotate = engine.plan( + changed, Blueprint.from_dict(document), scope="inventory", options=ApplyOptions(write_credentials=True) + ) + assert rotate.phase("inventory").status == "planned" + rotate.apply() + assert inventory.devices["device1"].configuration.config.cinfo.auth.password == "new-password" assert ( engine.plan(_device(), Blueprint.from_dict(document), scope="inventory").phase("inventory").status == "no_change" ) +def test_masked_secrets_on_read_do_not_cause_changes( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + device = _device( + credentials=Credentials(username="test-user", password="test-password"), + alternative_addresses=[ + AlternativeAddress( + address="192.0.2.21", credentials=Credentials(username="test-user", password="alt-password") + ) + ], + ) + engine.apply(device, _blueprint()) + # The server masks secrets on read. + cinfo = inventory.devices["device1"].configuration.config.cinfo + cinfo.auth.password = "********" + cinfo.altAddressesWithAuth = [{"address": "192.0.2.21", "authentication": {"user": "test-user", "password": ""}}] + + plan = engine.plan(device, _blueprint()) + assert plan.fully_resolved and not plan.has_changes + + # Any other managed change writes the supplied secrets along with it. + relabelled = device.model_copy(update={"label": "device-renamed"}) + engine.apply(relabelled, _blueprint(), scope="inventory") + cinfo = inventory.devices["device1"].configuration.config.cinfo + assert cinfo.auth.password == "test-password" + assert cinfo.altAddressesWithAuth[0]["authentication"]["password"] == "alt-password" + + def test_api_key_is_redacted_in_the_plan(engine: BlueprintEngine) -> None: secret = "synthetic-key-value" document = { @@ -328,6 +377,44 @@ def test_discovery_timeout_reports_created_id(engine: BlueprintEngine, server: F assert [name for name, _ in server.writes].count("add_devices") == 4 +def test_permanent_errors_during_discovery_fail_without_waiting( + engine: BlueprintEngine, server: FakeInspectServer +) -> None: + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + server.discoverable["device100"] = matrox_layout("device100", "tx") + wrong_mode = {"processor_type": "matrox.convertip.default", "params": {"mode": "rx"}} + with pytest.raises(BlueprintApplyError) as info: + engine.apply(device, _blueprint(vertex_processor=wrong_mode)) + result = info.value.result + assert result.status == "partial" and result.inventory_id == "device100" + assert "mode 'rx' was requested" in result.phase("topology").message + assert "not ready" not in result.phase("topology").message + assert engine._clock() == 0.0 + + +def test_interface_miss_is_final_only_once_ports_exist( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + with pytest.raises(BlueprintTargetError, match="no IP port") as info: + engine.plan(_device(), _blueprint(ip_vertex_mapping={"stream-a": ["missing"]})) + assert not isinstance(info.value, TopologyNotReadyError) + empty = ScopeData(device_id="device1", device=DeviceRecord(id="device1")) + with pytest.raises(TopologyNotReadyError, match="no IP port"): + resolve_interfaces({"stream-a": ["P1"]}, empty, None) + + +def test_address_conflicts_use_one_lookup_for_all_addresses(engine: BlueprintEngine, inventory: FakeInventory) -> None: + inventory.seed("device1", label="device-a", address="Device-A.example") + inventory.seed("device2", label="device-b", address="2001:db8:0:0::1") + new = BlueprintDevice( + key="key-c", label="device-c", management_address="device-a.example", alternative_addresses=["2001:db8::1"] + ) + with pytest.raises(BlueprintTargetError, match="'device-a.example' is used by device1; address '2001:db8::1'"): + engine.plan(new, _blueprint(), scope="inventory") + assert inventory.address_lookups == 1 + + def test_sync_policies(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: inventory.seed("device1", label="device-a", custom={"port": 8080}) with pytest.raises(BlueprintTargetError, match="not in the topology"): @@ -425,7 +512,9 @@ def test_inventory_preread_failure_is_a_known_rejection( ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" - inventory.fail_next_write = ValueError("Failed to retrieve existing device configuration from Inventory: timed out") + inventory.fail_next_write = InventoryWriteNotAppliedError( + "Failed to retrieve existing device configuration from Inventory: timed out", operation="update" + ) with pytest.raises(BlueprintApplyError) as info: engine.apply(_device(), _blueprint()) result = info.value.result @@ -434,6 +523,29 @@ def test_inventory_preread_failure_is_a_known_rejection( assert inventory.writes == [] +@pytest.mark.parametrize(("operation", "status"), [("add", "failed"), ("update", "unknown")]) +def test_inventory_create_rejection_outcomes( + engine: BlueprintEngine, inventory: FakeInventory, operation: str, status: str +) -> None: + # A rejected add is known; a rejected follow-up update means the record may already exist. + inventory.fail_next_write = InventoryWriteNotAppliedError("rejected", operation=operation) # type: ignore[arg-type] + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(BlueprintApplyError) as info: + engine.apply(device, _blueprint(), scope="inventory") + assert info.value.result.status == status + + +def test_untyped_inventory_value_error_is_an_unknown_outcome( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + inventory.fail_next_write = ValueError("Failed to update device in VideoIPath-Inventory.") + with pytest.raises(BlueprintApplyError) as info: + engine.apply(_device(), _blueprint()) + assert info.value.result.status == "unknown" + + # --- Modules --- diff --git a/tests/blueprints/test_processors.py b/tests/blueprints/test_processors.py index 5831cb6..af06d9e 100644 --- a/tests/blueprints/test_processors.py +++ b/tests/blueprints/test_processors.py @@ -23,6 +23,7 @@ ProcessorResult, SourceFacts, TagDelta, + TopologyNotReadyError, VertexProcessor, ) from videoipath_automation_tool.blueprints.inspect import InspectGateway, build_context @@ -181,17 +182,25 @@ def test_processing_is_deterministic_regardless_of_order() -> None: @pytest.mark.parametrize( - ("mutate", "message"), + ("mutate", "error", "message"), [ - (lambda s: s.vertex_forms.pop("device1.0.ar.v"), "unknown kind"), - (lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver 7"), "indices"), - (lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver"), "no 'Video Receiver ' index"), - (lambda s: _set_media(s, "device1.0.ar.v", "video"), "unsupported RX layout with 5 video and 0 audio"), + (lambda s: s.vertex_forms.pop("device1.0.ar.v"), TopologyNotReadyError, "unknown kind"), + (lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver 7"), ProcessorInputError, "indices"), + ( + lambda s: _relabel(s, "device1.0.vr3.v", "Video Receiver"), + ProcessorInputError, + "no 'Video Receiver ' index", + ), + ( + lambda s: _set_media(s, "device1.0.ar.v", "video"), + ProcessorInputError, + "unsupported RX layout with 5 video and 0 audio", + ), ], ) -def test_invalid_four_split_layouts(mutate: Any, message: str) -> None: +def test_invalid_four_split_layouts(mutate: Any, error: type[Exception], message: str) -> None: context = _context(matrox_layout("device1", "four_split"), mutate=mutate) - with pytest.raises(ProcessorInputError, match=message): + with pytest.raises(error, match=message): _process(context) @@ -212,7 +221,7 @@ def no_codecs(server: FakeInspectServer) -> None: for form in server.vertex_forms.values(): form["fields"]["typeFields"]["type"] = "ip" - with pytest.raises(ProcessorInputError, match="no codec vertices"): + with pytest.raises(TopologyNotReadyError, match="no codec vertices"): _process(_context(matrox_layout("device1", "tx"), mutate=no_codecs)) diff --git a/tests/inventory/__init__.py b/tests/inventory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/inventory/test_inventory_api.py b/tests/inventory/test_inventory_api.py new file mode 100644 index 0000000..0378eb2 --- /dev/null +++ b/tests/inventory/test_inventory_api.py @@ -0,0 +1,126 @@ +"""InventoryAPI address lookups and typed write rejections with a fake connector.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from videoipath_automation_tool.apps.inventory.app.app import InventoryApp +from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError +from videoipath_automation_tool.apps.inventory.inventory_api import InventoryAPI +from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice + +NMOS = "com.nevion.NMOS_multidevice-0.1.0" + + +class FakeRest: + def __init__(self, data: Any) -> None: + self._data = data + self.get_calls: list[str] = [] + + def get(self, url_path: str, **kwargs: Any) -> SimpleNamespace: + self.get_calls.append(url_path) + return SimpleNamespace(data=self._data) + + +class FakeRpc: + def __init__(self, status: str) -> None: + self._status = status + + def post(self, url_path: str, body: Any, **kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(header=SimpleNamespace(status=self._status)) + + +def _api(data: Any = None, *, rpc_status: str = "OK") -> InventoryAPI: + return InventoryAPI(SimpleNamespace(rest=FakeRest(data), rpc=FakeRpc(rpc_status))) # type: ignore[arg-type] + + +def _devices(*devices: dict[str, Any]) -> dict[str, Any]: + return {"config": {"devman": {"devices": {"_items": list(devices)}}}} + + +def _device(device_id: str, address: Any, alternates: Any = None) -> dict[str, Any]: + cinfo: dict[str, Any] = {"address": address} + if alternates is not None: + cinfo["altAddresses"] = alternates + return {"_id": device_id, "config": {"cinfo": cinfo}} + + +# --- Address lookups --- + + +def test_find_device_ids_by_addresses_normalizes_both_sides_in_one_read() -> None: + api = _api( + _devices( + _device("device1", "Device-A.example"), + _device("device2", "192.0.2.1", ["2001:db8:0:0::1"]), + _device("device3", "192.0.2.3", None), + ) + ) + found = api.find_device_ids_by_addresses(["device-a.example", "2001:db8::1", " 192.0.2.3 ", "192.0.2.9", ""]) + assert found == { + "device-a.example": ["device1"], + "2001:db8::1": ["device2"], + " 192.0.2.3 ": ["device3"], + "192.0.2.9": [], + } + assert len(api.vip_connector.rest.get_calls) == 1 + + +def test_find_device_ids_by_addresses_reports_every_match_sorted() -> None: + api = _api(_devices(_device("device2", "192.0.2.1"), _device("device1", "192.0.2.5", ["192.0.2.1"]))) + assert api.find_device_ids_by_addresses(["192.0.2.1"]) == {"192.0.2.1": ["device1", "device2"]} + + +@pytest.mark.parametrize("data", [None, [], {}, {"config": {"devman": {"devices": {}}}}]) +def test_find_device_ids_by_addresses_without_items_is_no_match(data: Any) -> None: + assert _api(data).find_device_ids_by_addresses(["192.0.2.1"]) == {"192.0.2.1": []} + + +def test_find_device_ids_by_addresses_without_input_skips_the_read() -> None: + api = _api(_devices(_device("device1", "192.0.2.1"))) + assert api.find_device_ids_by_addresses(["", ""]) == {} + assert api.vip_connector.rest.get_calls == [] + + +def test_get_device_id_by_address_keeps_exact_matching() -> None: + api = _api(_devices(_device("device1", "Device-A.example"), _device("device2", "192.0.2.1", ["192.0.2.2"]))) + assert api.get_device_id_by_address("Device-A.example") == "device1" + assert api.get_device_id_by_address("device-a.example") is None + assert api.get_device_id_by_address("192.0.2.2") == "device2" + with pytest.raises(ValueError, match="Response data is empty"): + _api(None).get_device_id_by_address("192.0.2.1") + + +# --- Write rejections --- + + +def test_rejected_add_is_typed() -> None: + with pytest.raises(InventoryWriteNotAppliedError, match="Failed to add device") as info: + _api(rpc_status="ERROR").add_device(InventoryDevice.create(NMOS), config_only=True) + assert info.value.operation == "add" + assert isinstance(info.value, ValueError) + + +def test_rejected_update_is_typed() -> None: + device = InventoryDevice.create(NMOS) + device.configuration.id = "device1" + with pytest.raises(InventoryWriteNotAppliedError, match="Failed to update device") as info: + _api(rpc_status="ERROR").update_device(device, config_only=True) + assert info.value.operation == "update" + + +def test_failed_preread_before_update_is_typed(monkeypatch: pytest.MonkeyPatch) -> None: + app = InventoryApp(SimpleNamespace(rest=FakeRest(None), rpc=FakeRpc("OK"))) # type: ignore[arg-type] + + def missing(**kwargs: Any) -> InventoryDevice: + raise ValueError("timed out") + + monkeypatch.setattr(app._inventory_api, "get_device", missing) + device = InventoryDevice.create(NMOS) + device.configuration.id = "device1" + with pytest.raises(InventoryWriteNotAppliedError, match="Failed to retrieve existing device") as info: + app.update_device(device, compare_config=False, config_only=True) + assert info.value.operation == "update" From d0560ad15a43d643b6c57eca4d3ba0f7c968e684 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:46:19 +0200 Subject: [PATCH 3/7] review --- docs/architecture/blueprints/concepts.md | 32 ++++-- .../decisions/002-plan-then-apply.md | 8 ++ .../blueprints/decisions/005-app-gateways.md | 11 +- docs/architecture/blueprints/reference.md | 52 ++++++--- docs/getting-started-guide/05_Blueprints.md | 9 +- .../apps/inventory/app/app.py | 2 +- .../apps/inventory/inventory_api.py | 8 +- .../blueprints/engine.py | 66 ++++++++--- .../blueprints/inspect.py | 16 ++- .../blueprints/inventory.py | 11 +- .../blueprints/models.py | 8 +- tests/blueprints/conftest.py | 6 + tests/blueprints/test_engine.py | 108 +++++++++++++++++- tests/inventory/test_inventory_api.py | 9 +- 14 files changed, 286 insertions(+), 60 deletions(-) diff --git a/docs/architecture/blueprints/concepts.md b/docs/architecture/blueprints/concepts.md index 9c74233..716e6e7 100644 --- a/docs/architecture/blueprints/concepts.md +++ b/docs/architecture/blueprints/concepts.md @@ -138,8 +138,10 @@ exact edits cannot be known yet, the topology phases are marked `deferred` and `fully_resolved` is `False`. That happens when: - this plan creates the Inventory record the topology device id will come from -- planned Inventory changes may alter discovery (address, alternate addresses, - credentials, generic settings, custom settings, or `active`) +- an Inventory update may alter discovery (address, alternate addresses, + credentials, generic settings, custom settings, or `active`). Apply stops + after Inventory, sets `replan_required`, and returns `status="partial"`. + Plan again once the driver has rediscovered the device - the device is not in the topology yet and `sync` allows adding it - synchronization is pending and this target is allowed to perform it @@ -169,23 +171,37 @@ replay in another process. 5. `module_tags` — `assignTag` / `unassignTag` actions, one tag at a time. These are not part of the topology commit. 6. `verification` — read back what this apply wrote. +An Inventory update that may alter discovery does not continue into discovery +or topology in the same apply. Those phases stay `deferred`, the result status +is `partial`, and `replan_required` is true. The call does not raise. Inventory +verification still runs. After the driver has rediscovered the device, build a +new plan. Creating a record still polls for discovery: a new device has no +previous topology. + `sync` defaults to `add_only`. `none` requires the device to already be in the topology. `add_only` adds the device and synchronizes new elements; if the pending sync would update or remove elements, apply fails instead of escalating. `reconcile` allows a full sync. The gateway calls `sync_devices` without a conflict strategy, so Inspect's default `ConflictStrategy.STRICT` applies. +A reported `syncDevices` failure is an error and is not retried. Adding a +device that is not discovered yet stays `TopologyNotReadyError` and is retried +until `discovery_timeout`. + A module target never reaches this add/sync path. Unchanged phases are skipped. An apply that would change nothing writes nothing. `dry_run=True` runs the same stale-plan, conflict, and pending-edit checks, then returns before any write. Deferred topology stays `deferred` on a dry run, -because those edits depend on writes the dry run does not perform. `status` is -`planned` when writes would have run, and `no_change` otherwise. - -Before an Inventory update, the gateway re-reads the record and compares -fingerprints of the managed fields. Before a topology commit that was fully -resolved at plan time, it re-reads the scope and compares the fingerprint, +because those edits depend on writes the dry run does not perform. A +topology-affecting update also stays `deferred`, with `replan_required` set. +`status` is `planned` when writes would have run, and `no_change` otherwise. + +Before an Inventory update, the gateway re-reads the record, compares +fingerprints of the managed fields, and repeats the label and address conflict +check. A response with no address item list fails the check. Before a topology +commit that was fully resolved at plan time, it re-reads the scope and compares +the fingerprint, including sibling endpoint labels that the collision check depends on. Module tags are re-read immediately before their phase. Pending uncommitted Inspect edits that overlap the plan's device, vertices, or module are rejected. diff --git a/docs/architecture/blueprints/decisions/002-plan-then-apply.md b/docs/architecture/blueprints/decisions/002-plan-then-apply.md index 711f440..35bdec6 100644 --- a/docs/architecture/blueprints/decisions/002-plan-then-apply.md +++ b/docs/architecture/blueprints/decisions/002-plan-then-apply.md @@ -34,6 +34,12 @@ for discovery, then computes the edits from the blueprint, parameters, naming, and target captured on the plan. A dry run does not invent those edits. It reports the phases as `deferred`. +An update of address, alternate addresses, credentials, generic or custom +settings, or `active` can change the topology the driver discovers. Apply does +not poll for that. It stops after Inventory, sets `replan_required`, and returns +`status="partial"`. The caller plans again after rediscovery. Creating a record +still waits for discovery, because a new device has no previous topology. + `dry_run=True` runs the stale-plan, conflict, and pending-edit checks and performs no write. @@ -46,6 +52,8 @@ performs no write. - A failure after Inventory create is `partial`. The result carries `inventory_id` so the caller can bind it and retry without creating a second record. +- A topology-affecting Inventory update is `partial` with `replan_required`. + The Inventory write stands. The next plan sees the updated record. - A write that raises, when the client cannot prove the server rejected it, is `unknown`. The caller reads the server back (a new plan does that) before retrying. diff --git a/docs/architecture/blueprints/decisions/005-app-gateways.md b/docs/architecture/blueprints/decisions/005-app-gateways.md index c3227dc..5033c48 100644 --- a/docs/architecture/blueprints/decisions/005-app-gateways.md +++ b/docs/architecture/blueprints/decisions/005-app-gateways.md @@ -25,13 +25,18 @@ record, resolves an SNMP configuration by exact label or id, creates with `find_device_id_by_label`. Address conflict checks use `find_device_ids_by_addresses`: one read for all addresses, compared after normalization on both sides (IP literals in compressed form, other identifiers -case-insensitively). An empty inventory response is no match. +case-insensitively). An empty item list is no match. A response with no item +list raises; the gateway turns that into `BlueprintError`. `InspectGateway` is the only object that calls `app.inspect`. Scope data is built from fresh collector and lookup reads (`get_device_detail`, `lookup_inspect_device`, `lookup_vertices`), not from the live snapshot's -domain objects. Pending edits are visible only through `staged_edit_keys()`, -which is used to reject overlap, not to seed the plan. +domain objects. If the collector already has the device, a failed edit-form +lookup is `BlueprintError`. The virtual-device fallback, where there is no +topology node, still treats a missing form as absent. Pending edits are visible +only through `staged_edit_keys()`, which is used to reject overlap, not to seed +the plan. A reported `syncDevices` failure is `BlueprintError` and is not +retried. `addDevices` failure stays `TopologyNotReadyError`. Topology writes go through one `InspectTransaction`: `update_device` and `update_vertex` intents, then `commit()`. Module tags use `assignTag` and diff --git a/docs/architecture/blueprints/reference.md b/docs/architecture/blueprints/reference.md index 9e264fb..f1a8d52 100644 --- a/docs/architecture/blueprints/reference.md +++ b/docs/architecture/blueprints/reference.md @@ -50,7 +50,7 @@ a file path string, a `pathlib.Path` (or another `os.PathLike[str]`), or a | `summary()` | Human-readable, redacted before/after listing, bindings, diagnostics, and unresolved work. | | `phases`, `phase(name)` | `PlannedPhase` records (`planned`, `no_change`, `deferred`, `skipped`) with their `PlannedOperation`s and `FieldChange`s. | | `has_changes` | `True` when any phase is `planned` or `deferred`. | -| `fully_resolved` | `False` when topology work is deferred until apply. | +| `fully_resolved` | `False` when topology phases are deferred until apply, or until a new plan after a topology-affecting Inventory update. | | `interface_bindings`, `diagnostics`, `skipped_sections` | Resolved `ip_vertex_mapping`, warnings, and sections the scope skipped. | | `source_key`, `scope`, `inventory_variant`, `topology_variant`, `inventory_id`, `topology_target`, `driver_id`, `driver_schema_version`, `processor_type`, `blueprint_digest` | Identity of what was planned. | @@ -62,11 +62,12 @@ replay elsewhere. | Field | Meaning | |---|---| -| `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. | +| `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. `partial` with `replan_required` is returned, not raised. | | `ok` | `True` for `succeeded`, `no_change`, and `planned`. | +| `replan_required` | `True` when an Inventory update may change discovery. Apply stopped after Inventory. `status` is `partial`, or `planned` on a dry run. Plan again after rediscovery. | | `inventory_id`, `topology_device_id`, `module_id`, `source_key` | Ids involved. Persist `inventory_id` in your source system. | -| `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. | -| `interface_bindings`, `diagnostics` | As on the plan, after deferred work was materialized. | +| `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. `deferred` also appears when a new plan is required, not only in a dry run. | +| `interface_bindings`, `diagnostics` | As on the plan, after deferred work was materialized. Empty when apply stopped for a replan. | | `materialized` | `True` when deferred topology work was computed during apply. | | `verification`, `verification_detail` | `confirmed`, `unconfirmed`, or `not_applicable`. | | `dry_run` | Whether this was a dry run. | @@ -413,16 +414,22 @@ it writes nothing. ### Deferred topology work Sometimes the topology edits depend on work that has not happened yet: a record -this plan creates, Inventory changes that may alter discovery (address, -alternative addresses, credentials, generic or custom settings, `active`), a -device that is not in the topology yet, or a synchronization that is still -pending. The plan marks that work `deferred`, and `plan.fully_resolved` is -`False`. During `apply()` the engine polls for discovery up to +this plan creates, a device that is not in the topology yet, or a synchronization +that is still pending. The plan marks that work `deferred`, and `plan.fully_resolved` +is `False`. During `apply()` the engine polls for discovery up to `discovery_timeout`, then computes the exact edits from the blueprint, parameters, naming, and target captured on the plan. Those edits are recorded on the result. If that later step fails, the earlier phases stay applied, and the result says so. +An Inventory update that changes address, alternate addresses, credentials, +generic or custom settings, or `active` is different. The plan marks the topology +phases `deferred` with `fully_resolved` false, but `apply()` stops after +Inventory. It does not poll, and it does not guess when rediscovery has finished. +The result has `status="partial"` and `replan_required=True`, and the call does +not raise. Inventory verification still runs. Plan again once the driver has +rediscovered the device. + If you want to see every write before it happens, split the scopes. Apply `scope="inventory"`, add the device with `app.inspect.add_devices_to_topology([...])`, then build a fully resolved `scope="topology"` plan, read it, and call @@ -432,9 +439,10 @@ then build a fully resolved `scope="topology"` plan, read it, and call `plan.apply(dry_run=True)` runs the same stale-plan, conflict, and pending-edit checks, then returns before any write. Deferred topology stays `deferred`, -because those edits depend on writes the dry run does not perform. `status` is -`planned` when writes would have run, and `no_change` otherwise. You can dry-run a -plan and apply that same plan afterwards. +because those edits depend on writes the dry run does not perform. A +topology-affecting update also stays `deferred`, with `replan_required` set. +`status` is `planned` when writes would have run, and `no_change` otherwise. You +can dry-run a plan and apply that same plan afterwards. ### Synchronization @@ -444,6 +452,10 @@ plan and apply that same plan afterwards. | `"add_only"` | Default. Adds the device and synchronizes new elements only. If that synchronization would update or remove elements, the apply fails. It does not escalate to a full reconcile. | | `"reconcile"` | A full synchronization is allowed. | +A `syncDevices` failure is an error and is not retried. Adding a device that is +not discovered yet stays `TopologyNotReadyError` and is retried until +`discovery_timeout`. + Service conflicts fail the apply. The engine does not invalidate or cancel services. A module target does not add its parent device to the topology and does not synchronize it. If the parent is absent, planning fails; if it still @@ -453,7 +465,9 @@ needs a sync, the plan reports a diagnostic. Before writing, the engine reads the Inventory record and the scoped topology again. If anything the plan relied on has changed, it raises -`BlueprintConflictError`. Build a new plan. The Inspect transaction then checks +`BlueprintConflictError`. An Inventory update also repeats the label and address +conflict check. A response with no address item list fails that check. Build a +new plan. The Inspect transaction then checks its own baseline, and module tags are read again immediately before their phase. These checks run in the client, so a short gap between the read and the write remains. Uncommitted edits on `app.inspect` that overlap the plan are rejected, @@ -462,11 +476,12 @@ are. ## 8. Results, errors, and recovery -`plan.apply()` returns an `ApplyResult` when `status` is `succeeded` or -`no_change` (`planned` on a dry run). On failure it raises `BlueprintApplyError`. -The exception's `.result` has `status` `failed` (nothing was written), `partial` -(something was written), or `unknown` (a write raised and the client cannot prove -the server rejected it). The original error is `__cause__`. +`plan.apply()` returns an `ApplyResult` when `status` is `succeeded`, +`no_change`, or `partial` with `replan_required` (`planned` on a dry run). On +failure it raises `BlueprintApplyError`. The exception's `.result` has `status` +`failed` (nothing was written), `partial` (something was written), or `unknown` +(a write raised and the client cannot prove the server rejected it). The +original error is `__cause__`. ```python from videoipath_automation_tool.blueprints import BlueprintApplyError @@ -482,6 +497,7 @@ except BlueprintApplyError as error: What you do next depends on how far the apply got: +- An Inventory update may change the discovered topology. `status` is `partial` and `replan_required` is true. Apply did not raise. Plan again after rediscovery. - The record was created, then discovery timed out. `status` is `partial` and `inventory_id` is set. Retry with `BlueprintDevice(..., inventory_id=result.inventory_id)`. - Topology committed, then a module tag failed. `status` is `partial`. The result lists the committed edits and the tag operations that finished. Replan to finish the rest. - A write timed out. `status` is `unknown`. Read the server back before you retry. Building a new plan does that read. diff --git a/docs/getting-started-guide/05_Blueprints.md b/docs/getting-started-guide/05_Blueprints.md index e68dae6..53aa84a 100644 --- a/docs/getting-started-guide/05_Blueprints.md +++ b/docs/getting-started-guide/05_Blueprints.md @@ -145,7 +145,14 @@ Apply runs these phases in order, skipping any with nothing to do: When the plan creates the Inventory record, the topology edits cannot be known yet. The plan marks them `deferred` and computes them during apply, after -discovery. If the server changed since planning, apply raises +discovery. + +When an update changes the address, alternate addresses, credentials, generic or +custom settings, or whether the device is active, apply stops after Inventory. +The result is `partial` with `replan_required`. Plan again after the driver has +rediscovered the device. + +If the server changed since planning, apply raises `BlueprintConflictError`; build a new plan. Behavior is tuned with `ApplyOptions` (`sync`, `discovery_timeout`, ...). diff --git a/src/videoipath_automation_tool/apps/inventory/app/app.py b/src/videoipath_automation_tool/apps/inventory/app/app.py index ee0cfd8..8de76f6 100644 --- a/src/videoipath_automation_tool/apps/inventory/app/app.py +++ b/src/videoipath_automation_tool/apps/inventory/app/app.py @@ -239,7 +239,7 @@ def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> dict[str, li """Map each of ``addresses`` to the sorted ids of devices whose management or alternate address matches it (IP literals normalized, other identifiers case-insensitive). - One server read for all addresses. An empty inventory response is no match. + One server read for all addresses. A response with no item list raises ``ValueError``. """ return self._inventory_api.find_device_ids_by_addresses(addresses) diff --git a/src/videoipath_automation_tool/apps/inventory/inventory_api.py b/src/videoipath_automation_tool/apps/inventory/inventory_api.py index b09a37d..f64c722 100644 --- a/src/videoipath_automation_tool/apps/inventory/inventory_api.py +++ b/src/videoipath_automation_tool/apps/inventory/inventory_api.py @@ -593,14 +593,16 @@ def find_device_ids_by_addresses(self, addresses: Iterable[str]) -> dict[str, li Addresses are compared after :func:`normalize_address` on both sides (IP literals in compressed form, other identifiers case-insensitively). One bulk read for all addresses. Empty addresses - are ignored; a missing item list is no match. + are ignored. A response with no item list raises ``ValueError``. """ wanted = {address: normalize_address(address) for address in addresses if address} if not wanted: return {} + index = self._read_device_addresses() + if index is None: + raise ValueError("Response data is empty.") normalized_index = [ - (device_id, {normalize_address(known) for known in known_addresses}) - for device_id, known_addresses in self._read_device_addresses() or [] + (device_id, {normalize_address(known) for known in known_addresses}) for device_id, known_addresses in index ] return { address: sorted(device_id for device_id, known in normalized_index if normalized in known) diff --git a/src/videoipath_automation_tool/blueprints/engine.py b/src/videoipath_automation_tool/blueprints/engine.py index f8c0352..59d899f 100644 --- a/src/videoipath_automation_tool/blueprints/engine.py +++ b/src/videoipath_automation_tool/blueprints/engine.py @@ -6,6 +6,11 @@ ``engine.apply(...)`` is ``engine.plan(...).apply()``. ``dry_run=True`` runs the same execution path — including the stale-plan, conflict, and staged-edit checks — but performs no write. +A topology-affecting Inventory update (address, alternate addresses, credentials, generic or custom +settings, or ``active``) stops after Inventory. The result is ``partial`` with ``replan_required``; +plan again after the driver has rediscovered the device. Creating a record still waits for discovery +during apply, because a new device has no previous topology. + Execution phases (absent or unchanged phases are skipped): Inventory → discovery readiness → topology membership/synchronization → topology commit (one ``InspectTransaction``) → module tags (separate RPCs) → verification. Inventory and Inspect writes are separate operations; there is no @@ -98,9 +103,10 @@ def inspect(self) -> Any: class BlueprintPlan(BaseModel): """A reviewed, immutable preview of intended changes. ``apply()`` executes exactly this plan. - ``fully_resolved`` is ``False`` when topology work depends on earlier phases (Inventory creation, - topology-affecting Inventory changes, topology membership, or pending synchronization); that work - is materialized during ``apply()`` from the captured configuration and recorded in the result. + ``fully_resolved`` is ``False`` when topology work depends on earlier phases. Inventory creation, + topology membership, and pending synchronization are materialized during ``apply()`` from the + captured configuration. A topology-affecting Inventory update is not: ``apply()`` stops after + Inventory and the result asks for a new plan. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -136,8 +142,12 @@ def apply(self, *, dry_run: bool = False) -> ApplyResult: With ``dry_run=True`` every check runs against current server state, but no write, topology action, or tag action is performed; the result reports the writes that would run - (``status="planned"``). Deferred topology work needs earlier writes and is reported as + (``status="planned"``). Deferred topology work that still needs earlier writes is reported as ``deferred``. + + A topology-affecting Inventory update stops after Inventory on a real run (``status="partial"``, + ``replan_required=True``) and does not raise. A dry run of that plan reports the same phases as + ``deferred`` and sets ``replan_required``. """ return self._executor.execute(self, dry_run=dry_run) @@ -163,10 +173,16 @@ def summary(self) -> str: for diagnostic in self.diagnostics: lines.append(f"[{diagnostic.level}] {diagnostic.code}: {diagnostic.message}") if not self.fully_resolved: - lines.append( - "Not fully resolved: deferred topology work is materialized during apply() after earlier phases; " - "if it then fails validation, earlier phases remain applied and are reported." - ) + if self._captured.topology_requires_replan: + lines.append( + "Not fully resolved: apply stops after Inventory; replan once the driver has rediscovered " + "the device." + ) + else: + lines.append( + "Not fully resolved: deferred topology work is materialized during apply() after earlier phases; " + "if it then fails validation, earlier phases remain applied and are reported." + ) return "\n".join(lines) def __repr__(self) -> str: @@ -370,6 +386,7 @@ class _Captured(BaseModel): topology_target: TopologyTarget | None = None topology_work: TopologyWork | None = None topology_deferred: bool = False + topology_requires_replan: bool = False @property def source(self) -> SourceFacts: @@ -401,7 +418,9 @@ def build(self) -> BlueprintPlan: diagnostics: list[Diagnostic] = [] inventory_work, current = self._plan_inventory(phases) - target, deferred_reason, topology_work = self._plan_topology(inventory_work, current, phases, diagnostics) + target, deferred_reason, topology_work, topology_requires_replan = self._plan_topology( + inventory_work, current, phases, diagnostics + ) captured = captured.model_copy( update={ @@ -409,6 +428,7 @@ def build(self) -> BlueprintPlan: "topology_target": target, "topology_work": topology_work, "topology_deferred": deferred_reason is not None, + "topology_requires_replan": topology_requires_replan, } ) plan = BlueprintPlan( @@ -469,13 +489,13 @@ def _plan_topology( current: InventoryDevice | None, phases: list[PlannedPhase], diagnostics: list[Diagnostic], - ) -> tuple[TopologyTarget | None, str | None, TopologyWork | None]: + ) -> tuple[TopologyTarget | None, str | None, TopologyWork | None, bool]: captured = self._captured resolved = captured.resolved if resolved.topology is None: reason = resolved.skipped.get("topology") phases.extend(PlannedPhase(name=name, status="skipped", reason=reason) for name in _TOPOLOGY_PHASES) - return None, None, None + return None, None, None, False device = captured.device target: TopologyTarget | None = device.topology @@ -488,8 +508,15 @@ def _plan_topology( ) reason: str | None = None + requires_replan = False if target is None: reason = "the topology device is the Inventory record created by this plan" + elif inventory_work is not None and inventory_work.affects_topology and inventory_work.action == "update": + reason = ( + "planned Inventory changes may alter the discovered topology; " + "apply stops after Inventory — plan again once the driver has rediscovered the device" + ) + requires_replan = True elif inventory_work is not None and inventory_work.affects_topology: reason = "planned Inventory changes may alter the discovered topology" else: @@ -497,7 +524,7 @@ def _plan_topology( if reason is not None: phases.extend(PlannedPhase(name=name, status="deferred", reason=reason) for name in _TOPOLOGY_PHASES) - return target, reason, None + return target, reason, None, requires_replan assert target is not None work = self._materialize(target, current) @@ -518,7 +545,7 @@ def _plan_topology( operations=work.module_operations, ) ) - return target, None, work + return target, None, work, False def _membership_reason(self, target: TopologyTarget, diagnostics: list[Diagnostic]) -> str | None: sync = self._captured.options.sync @@ -669,6 +696,7 @@ def _run_inventory(self) -> None: assert work.inventory_id is not None fresh = gateway.read(work.inventory_id) check_baseline(work, fresh) + recheck_conflicts(gateway, self._captured.device, work) if self._dry_run: self._planned(phase, [work.operation]) return @@ -683,6 +711,16 @@ def _run_inventory(self) -> None: def _run_topology(self) -> None: captured = self._captured + if captured.topology_requires_replan: + reason = self._plan.phase("topology").reason if self._plan.phase("topology") else None + for name in _TOPOLOGY_PHASES: + phase = self._enter(name) + phase.status = "deferred" + phase.message = reason + self._result.replan_required = True + if self._dry_run: + self._would_write = True + return if captured.topology_deferred and self._dry_run: reason = self._plan.phase("topology").reason if self._plan.phase("topology") else None for name in _TOPOLOGY_PHASES: @@ -894,6 +932,8 @@ def _finish(self) -> None: if self._dry_run: result.verification_detail = "dry run: nothing was written" result.status = "planned" if self._would_write else "no_change" + elif result.replan_required: + result.status = "partial" else: result.status = "succeeded" if self._wrote else "no_change" self._executor._logger.info("Blueprint apply for '%s' %s.", result.source_key, result.status) diff --git a/src/videoipath_automation_tool/blueprints/inspect.py b/src/videoipath_automation_tool/blueprints/inspect.py index ab628dc..37918d0 100644 --- a/src/videoipath_automation_tool/blueprints/inspect.py +++ b/src/videoipath_automation_tool/blueprints/inspect.py @@ -150,7 +150,7 @@ def _api(self) -> Any: def in_topology(self, device_id: str) -> bool: if self._detail(device_id) is not None: return True - return device_id.startswith("virtual.") and self._device_form(device_id) is not None + return device_id.startswith("virtual.") and self._device_form(device_id, required=False) is not None def sync_info(self, device_id: str) -> Any | None: """Pending sync for ``device_id``, or ``None`` when the server reports no sync record. @@ -171,7 +171,7 @@ def read_scope(self, target: TopologyTarget) -> ScopeData: device_id = target.device_id module_id = target.module_id if isinstance(target, ModuleTarget) else None node = self._detail(device_id) - form = self._device_form(device_id) + form = self._device_form(device_id, required=node is not None) if node is None and form is None: raise BlueprintTargetError(f"Inspect device '{device_id}' is not in the topology.") @@ -248,7 +248,7 @@ def add_to_topology(self, device_id: str) -> None: def sync(self, device_id: str, *, add_only: bool) -> None: if not self._app.sync_devices([device_id], add_only=add_only): - raise TopologyNotReadyError(f"syncDevices reported failure for '{device_id}'.") + raise BlueprintError(f"syncDevices failed for '{device_id}'; see the server log / Inspect sync dialog.") def commit(self, work: TopologyWork) -> Any: with self._app.transaction() as tx: @@ -277,11 +277,15 @@ def _detail(self, device_id: str) -> InspectApiNodeStatusItem | None: node = self._api.get_device_detail(device_id.replace(".", "-")) return node - def _device_form(self, device_id: str) -> Any | None: + def _device_form(self, device_id: str, *, required: bool) -> Any | None: try: return self._api.lookup_inspect_device(device_id).data.fields - except Exception: # noqa: BLE001 - return None + except Exception as exc: + if not required: + return None + raise BlueprintError( + f"Could not read the Inspect edit form of device '{device_id}': {type(exc).__name__}: {exc}" + ) from exc def _vertex_forms(self, vertex_ids: list[str]) -> dict[str, Any]: if not vertex_ids: diff --git a/src/videoipath_automation_tool/blueprints/inventory.py b/src/videoipath_automation_tool/blueprints/inventory.py index eea078d..92f6163 100644 --- a/src/videoipath_automation_tool/blueprints/inventory.py +++ b/src/videoipath_automation_tool/blueprints/inventory.py @@ -29,6 +29,7 @@ from videoipath_automation_tool.blueprints.errors import ( BlueprintCapabilityError, BlueprintConflictError, + BlueprintError, BlueprintTargetError, BlueprintValidationError, ValidationIssue, @@ -107,10 +108,16 @@ def ids_with_label(self, label: str) -> list[str]: return _as_list(found) def ids_by_addresses(self, addresses: list[str]) -> dict[str, list[str]]: - """Device ids per address (normalized comparison), in one read.""" + """Device ids per address (normalized comparison), in one read. + + A response with no item list raises :class:`BlueprintError`. + """ if not addresses: return {} - return self._app.find_device_ids_by_addresses(addresses) + try: + return self._app.find_device_ids_by_addresses(addresses) + except ValueError as exc: + raise BlueprintError(f"Could not read Inventory addresses for the conflict check: {exc}") from exc def resolve_snmp(self, reference: str | CatalogId) -> str: """Exact SNMP configuration id for a label or ``{id: ...}``; missing/ambiguous fail.""" diff --git a/src/videoipath_automation_tool/blueprints/models.py b/src/videoipath_automation_tool/blueprints/models.py index dd6e2a4..b48e40d 100644 --- a/src/videoipath_automation_tool/blueprints/models.py +++ b/src/videoipath_automation_tool/blueprints/models.py @@ -521,7 +521,7 @@ class InterfaceBinding(BaseModel): class PhaseResult(BaseModel): name: PhaseName status: Literal["completed", "no_change", "skipped", "failed", "unknown", "not_run", "planned", "deferred"] - """``planned`` / ``deferred`` occur only in dry runs (writes that would run / work that needs earlier writes).""" + """``planned`` occurs only in dry runs. ``deferred`` also appears when a new plan is required.""" message: str | None = None operations: list[PlannedOperation] = Field(default_factory=list) @@ -531,10 +531,16 @@ class ApplyResult(BaseModel): In a dry run (``dry_run=True``) nothing is written: ``status`` is ``planned`` when writes would run and ``no_change`` otherwise, and write phases report ``planned`` / ``deferred``. + + ``replan_required`` means a topology-affecting Inventory update finished and apply stopped. + ``status`` is ``partial`` (``planned`` on a dry run) and the call does not raise. Plan again + after the driver has rediscovered the device. """ status: Literal["succeeded", "no_change", "failed", "partial", "unknown", "planned"] = "no_change" dry_run: bool = False + replan_required: bool = False + """True when apply stopped after a topology-affecting Inventory update and a new plan is required.""" source_key: str inventory_id: str | None = None topology_device_id: str | None = None diff --git a/tests/blueprints/conftest.py b/tests/blueprints/conftest.py index 546fa19..1bf5a02 100644 --- a/tests/blueprints/conftest.py +++ b/tests/blueprints/conftest.py @@ -42,6 +42,7 @@ def __init__(self) -> None: self.snmp: dict[str, str] = {"default": "Default configuration", "snmp-1": "snmp-a"} self.writes: list[tuple[str, str]] = [] self.address_lookups = 0 + self.fail_address_read = False self.fail_next_write: Exception | None = None self._next_id = 100 @@ -92,6 +93,8 @@ def get_global_snmp_config_label_by_id(self, snmp_config_id: str) -> str | None: def find_device_ids_by_addresses(self, addresses: Any) -> dict[str, list[str]]: self.address_lookups += 1 + if self.fail_address_read: + raise ValueError("Response data is empty.") result: dict[str, list[str]] = {} for address in addresses: wanted = normalize_address(address) @@ -132,6 +135,7 @@ def __init__(self) -> None: self.fail_commit: Exception | None = None self.fail_tag: str | None = None self.fail_sync_info: Exception | None = None + self.fail_sync = False # Setup @@ -208,6 +212,8 @@ def add_devices(self, items: list[Any]) -> InspectApiSimpleActionResponse: def sync_devices(self, device_ids: list[str], add_only: bool = True, conflict_strategy: int = 0) -> Any: for device_id in device_ids: self.writes.append(("sync_devices", (device_id, add_only))) + if self.fail_sync: + return _action(False, ["sync rejected"]) pending = self.discoverable.pop(device_id, None) if pending is not None: self.install(pending) diff --git a/tests/blueprints/test_engine.py b/tests/blueprints/test_engine.py index b7ec0aa..8afcf39 100644 --- a/tests/blueprints/test_engine.py +++ b/tests/blueprints/test_engine.py @@ -276,6 +276,7 @@ def test_masked_secrets_on_read_do_not_cause_changes( ) ], ) + engine.apply(device, _blueprint(), scope="inventory") engine.apply(device, _blueprint()) # The server masks secrets on read. cinfo = inventory.devices["device1"].configuration.config.cinfo @@ -365,6 +366,53 @@ def test_create_discover_and_configure( assert server.vertex_forms["device100.0.vs.v"]["fields"]["label"] == "device-new-TX-video-01" +def _topology_affecting_document() -> Blueprint: + return Blueprint.from_dict( + {**MATROX, "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}}} + ) + + +def test_topology_affecting_update_stops_for_replan( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + document = _topology_affecting_document() + plan = engine.plan(_device(), document) + assert not plan.fully_resolved + assert plan.phase("topology").status == "deferred" + assert "plan again once the driver has rediscovered" in (plan.phase("topology").reason or "") + assert "stops after Inventory; replan" in plan.summary() + assert "materialized during apply()" not in plan.summary() + + result = plan.apply() + assert result.status == "partial" and result.replan_required and not result.ok + assert result.phase("inventory").status == "completed" and result.verification == "confirmed" + assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { + "deferred" + } + assert inventory.writes == [("update", "device1")] and server.writes == [] + assert inventory.devices["device1"].configuration.config.customSettings.disable_rx_sdp is True + + follow_up = engine.plan(_device(), document) + assert follow_up.fully_resolved and follow_up.phase("topology").status == "planned" + applied = follow_up.apply() + assert applied.status == "succeeded" and not applied.replan_required + assert "update_topology" in [name for name, _ in server.writes] + + +def test_topology_affecting_update_dry_run_requires_replan( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + result = engine.apply(_device(), _topology_affecting_document(), dry_run=True) + assert result.status == "planned" and result.replan_required and result.dry_run + assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { + "deferred" + } + assert "plan again" in (result.phase("topology").message or "") + assert inventory.writes == [] and server.writes == [] + + def test_discovery_timeout_reports_created_id(engine: BlueprintEngine, server: FakeInspectServer) -> None: device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") with pytest.raises(BlueprintApplyError) as info: @@ -415,6 +463,14 @@ def test_address_conflicts_use_one_lookup_for_all_addresses(engine: BlueprintEng assert inventory.address_lookups == 1 +def test_address_read_failure_fails_planning(engine: BlueprintEngine, inventory: FakeInventory) -> None: + inventory.fail_address_read = True + device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(BlueprintError, match="Could not read Inventory addresses") as info: + engine.plan(device, _blueprint(), scope="inventory") + assert type(info.value) is BlueprintError + + def test_sync_policies(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: inventory.seed("device1", label="device-a", custom={"port": 8080}) with pytest.raises(BlueprintTargetError, match="not in the topology"): @@ -441,6 +497,31 @@ def test_sync_lookup_failure_fails_planning( assert type(info.value) is BlueprintError +def test_sync_failure_is_not_retried( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + server.sync["device1"] = {"add": {"modules": 1}, "update": {}, "remove": {}} + server.fail_sync = True + with pytest.raises(BlueprintApplyError) as info: + engine.apply(_device(), _blueprint()) + result = info.value.result + assert result.status == "failed" + assert [name for name, _ in server.writes] == ["sync_devices"] + assert "syncDevices failed" in (result.phase("topology_sync").message or "") + assert engine._clock() == 0.0 + + +def test_device_form_lookup_failure_fails_planning( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + del server.device_forms["device1"] + with pytest.raises(BlueprintError, match="Inspect edit form") as info: + engine.plan(_device(), _blueprint()) + assert type(info.value) is BlueprintError + + # --- Conflicts and staged edits --- @@ -462,6 +543,21 @@ def test_stale_plans_are_rejected(engine: BlueprintEngine, inventory: FakeInvent assert server.writes == [] +def test_inventory_update_rechecks_label_conflicts( + engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.config.desc.label = "old" + plan = engine.plan(_device(), _blueprint()) + assert plan.fully_resolved and plan.phase("inventory").status == "planned" + inventory.seed("device2", label="device-a") + with pytest.raises(BlueprintApplyError) as info: + plan.apply() + assert info.value.result.status == "failed" + assert isinstance(info.value.__cause__, BlueprintTargetError) + assert inventory.writes == [] + + def test_staged_edits_overlap_is_rejected_and_unrelated_edits_survive( engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: @@ -920,8 +1016,16 @@ def test_module_with_own_inventory_record_never_touches_the_parent( ) plan = engine.plan(module, _blueprint(device=None)) assert plan.phase("inventory").status == "planned" and plan.phase("topology").status == "deferred" - result = plan.apply() - assert result.status == "succeeded" and result.materialized + assert "stops after Inventory" in (plan.phase("topology").reason or "") + stopped = plan.apply() + assert stopped.status == "partial" and stopped.replan_required and not stopped.materialized + assert inventory.writes == [("update", "device5")] + assert server.writes == [] + + follow_up = engine.plan(module, _blueprint(device=None)) + assert follow_up.fully_resolved + result = follow_up.apply() + assert result.status == "succeeded" and not result.replan_required assert inventory.writes == [("update", "device5")] assert not [name for name, _ in server.writes if name in ("add_devices", "sync_devices")] delta = next(payload for name, payload in server.writes if name == "update_topology") diff --git a/tests/inventory/test_inventory_api.py b/tests/inventory/test_inventory_api.py index 0378eb2..e4b1c90 100644 --- a/tests/inventory/test_inventory_api.py +++ b/tests/inventory/test_inventory_api.py @@ -75,8 +75,13 @@ def test_find_device_ids_by_addresses_reports_every_match_sorted() -> None: @pytest.mark.parametrize("data", [None, [], {}, {"config": {"devman": {"devices": {}}}}]) -def test_find_device_ids_by_addresses_without_items_is_no_match(data: Any) -> None: - assert _api(data).find_device_ids_by_addresses(["192.0.2.1"]) == {"192.0.2.1": []} +def test_find_device_ids_by_addresses_without_items_raises(data: Any) -> None: + with pytest.raises(ValueError, match="Response data is empty"): + _api(data).find_device_ids_by_addresses(["192.0.2.1"]) + + +def test_find_device_ids_by_addresses_with_empty_item_list_returns_no_match() -> None: + assert _api(_devices()).find_device_ids_by_addresses(["192.0.2.1"]) == {"192.0.2.1": []} def test_find_device_ids_by_addresses_without_input_skips_the_read() -> None: From ee0fa8776c5ca4bb6ed2097242134c8cf22a04f7 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:01:45 +0200 Subject: [PATCH 4/7] refinements --- .gitignore | 3 +- AGENTS.md | 12 +- docs/architecture/blueprints/README.md | 41 - docs/architecture/provisioning/README.md | 78 + .../{blueprints => provisioning}/concepts.md | 105 +- .../decisions/001-standalone-feature.md | 16 +- .../decisions/002-plan-then-apply.md | 14 +- .../decisions/003-managed-fields.md | 0 .../decisions/004-processors-propose.md | 10 +- .../decisions/005-app-gateways.md | 6 +- .../decisions/006-variant-merge.md | 9 +- .../decisions/007-typed-inputs.md | 51 + .../decisions/008-staged-readiness.md | 72 + .../decisions/009-external-connections.md | 47 + .../decisions/README.md | 7 +- .../processors.md | 24 +- .../{blueprints => provisioning}/reference.md | 491 +++- .../07_blueprints/matrox-convertip.yml | 41 - .../01_onboard_with_blueprint.py | 57 +- .../02_custom_processor_and_naming.py | 28 +- .../03_external_connections.py | 55 + .../07_provisioning/external-connections.yml | 17 + .../07_provisioning/matrox-convertip.yml | 51 + docs/examples/README.md | 13 +- docs/getting-started-guide/05_Blueprints.md | 195 -- docs/getting-started-guide/05_Provisioning.md | 239 ++ docs/getting-started-guide/README.md | 2 +- .../apps/inspect/api/inspect_api.py | 3 + .../apps/inventory/errors.py | 6 +- .../apps/inventory/inventory_api.py | 7 +- .../blueprints/resolution.py | 509 ---- .../schemas/blueprint-v1.schema.json | 1159 --------- .../{blueprints => provisioning}/__init__.py | 76 +- .../provisioning/connections.py | 279 +++ .../{blueprints => provisioning}/engine.py | 492 +++- .../{blueprints => provisioning}/errors.py | 47 +- .../provisioning/inputs.py | 163 ++ .../{blueprints => provisioning}/inspect.py | 274 ++- .../{blueprints => provisioning}/inventory.py | 57 +- .../{blueprints => provisioning}/loader.py | 6 +- .../{blueprints => provisioning}/models.py | 216 +- .../{blueprints => provisioning}/naming.py | 70 +- .../processors/__init__.py | 22 +- .../processors/matrox.py | 10 +- .../provisioning/resolution.py | 601 +++++ .../schemas/blueprint-v1.schema.json | 2142 +++++++++++++++++ .../provisioning/templates.py | 147 ++ src/vipat_cli_scripts/test_runner.py | 2 +- tests/e2e/conftest.py | 9 +- tests/e2e/helpers.py | 9 + tests/e2e/provisioning/__init__.py | 1 + tests/e2e/provisioning/conftest.py | 33 + .../e2e/provisioning/fixtures/mock-router.yml | 49 + tests/e2e/provisioning/helpers.py | 160 ++ tests/e2e/provisioning/test_connections.py | 157 ++ tests/e2e/provisioning/test_lifecycle.py | 177 ++ tests/e2e/provisioning/test_recovery.py | 199 ++ tests/inspect/test_api.py | 23 +- tests/inventory/test_inventory_api.py | 25 +- .../{blueprints => provisioning}/__init__.py | 0 .../{blueprints => provisioning}/conftest.py | 100 +- .../test_blueprint_inputs.py | 33 +- tests/provisioning/test_connection_inputs.py | 159 ++ tests/provisioning/test_connections.py | 450 ++++ .../test_engine.py | 486 ++-- tests/provisioning/test_live_regressions.py | 91 + .../test_loader.py | 22 +- .../test_naming.py | 10 +- .../test_processors.py | 16 +- tests/provisioning/test_readiness.py | 409 ++++ .../test_resolution.py | 141 +- tests/provisioning/test_v2.py | 379 +++ tests/test_test_runner.py | 75 + 73 files changed, 8359 insertions(+), 2826 deletions(-) delete mode 100644 docs/architecture/blueprints/README.md create mode 100644 docs/architecture/provisioning/README.md rename docs/architecture/{blueprints => provisioning}/concepts.md (64%) rename docs/architecture/{blueprints => provisioning}/decisions/001-standalone-feature.md (65%) rename docs/architecture/{blueprints => provisioning}/decisions/002-plan-then-apply.md (85%) rename docs/architecture/{blueprints => provisioning}/decisions/003-managed-fields.md (100%) rename docs/architecture/{blueprints => provisioning}/decisions/004-processors-propose.md (85%) rename docs/architecture/{blueprints => provisioning}/decisions/005-app-gateways.md (92%) rename docs/architecture/{blueprints => provisioning}/decisions/006-variant-merge.md (87%) create mode 100644 docs/architecture/provisioning/decisions/007-typed-inputs.md create mode 100644 docs/architecture/provisioning/decisions/008-staged-readiness.md create mode 100644 docs/architecture/provisioning/decisions/009-external-connections.md rename docs/architecture/{blueprints => provisioning}/decisions/README.md (66%) rename docs/architecture/{blueprints => provisioning}/processors.md (90%) rename docs/architecture/{blueprints => provisioning}/reference.md (53%) delete mode 100644 docs/examples/07_blueprints/matrox-convertip.yml rename docs/examples/{07_blueprints => 07_provisioning}/01_onboard_with_blueprint.py (50%) rename docs/examples/{07_blueprints => 07_provisioning}/02_custom_processor_and_naming.py (82%) create mode 100644 docs/examples/07_provisioning/03_external_connections.py create mode 100644 docs/examples/07_provisioning/external-connections.yml create mode 100644 docs/examples/07_provisioning/matrox-convertip.yml delete mode 100644 docs/getting-started-guide/05_Blueprints.md create mode 100644 docs/getting-started-guide/05_Provisioning.md delete mode 100644 src/videoipath_automation_tool/blueprints/resolution.py delete mode 100644 src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json rename src/videoipath_automation_tool/{blueprints => provisioning}/__init__.py (50%) create mode 100644 src/videoipath_automation_tool/provisioning/connections.py rename src/videoipath_automation_tool/{blueprints => provisioning}/engine.py (67%) rename src/videoipath_automation_tool/{blueprints => provisioning}/errors.py (69%) create mode 100644 src/videoipath_automation_tool/provisioning/inputs.py rename src/videoipath_automation_tool/{blueprints => provisioning}/inspect.py (76%) rename src/videoipath_automation_tool/{blueprints => provisioning}/inventory.py (90%) rename src/videoipath_automation_tool/{blueprints => provisioning}/loader.py (97%) rename src/videoipath_automation_tool/{blueprints => provisioning}/models.py (73%) rename src/videoipath_automation_tool/{blueprints => provisioning}/naming.py (83%) rename src/videoipath_automation_tool/{blueprints => provisioning}/processors/__init__.py (94%) rename src/videoipath_automation_tool/{blueprints => provisioning}/processors/matrox.py (97%) create mode 100644 src/videoipath_automation_tool/provisioning/resolution.py create mode 100644 src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json create mode 100644 src/videoipath_automation_tool/provisioning/templates.py create mode 100644 tests/e2e/provisioning/__init__.py create mode 100644 tests/e2e/provisioning/conftest.py create mode 100644 tests/e2e/provisioning/fixtures/mock-router.yml create mode 100644 tests/e2e/provisioning/helpers.py create mode 100644 tests/e2e/provisioning/test_connections.py create mode 100644 tests/e2e/provisioning/test_lifecycle.py create mode 100644 tests/e2e/provisioning/test_recovery.py rename tests/{blueprints => provisioning}/__init__.py (100%) rename tests/{blueprints => provisioning}/conftest.py (79%) rename tests/{blueprints => provisioning}/test_blueprint_inputs.py (75%) create mode 100644 tests/provisioning/test_connection_inputs.py create mode 100644 tests/provisioning/test_connections.py rename tests/{blueprints => provisioning}/test_engine.py (69%) create mode 100644 tests/provisioning/test_live_regressions.py rename tests/{blueprints => provisioning}/test_loader.py (71%) rename tests/{blueprints => provisioning}/test_naming.py (94%) rename tests/{blueprints => provisioning}/test_processors.py (95%) create mode 100644 tests/provisioning/test_readiness.py rename tests/{blueprints => provisioning}/test_resolution.py (55%) create mode 100644 tests/provisioning/test_v2.py create mode 100644 tests/test_test_runner.py diff --git a/.gitignore b/.gitignore index 5a31506..f5a149d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ __pycache__/ __intern/ *.egg-info/ .DS_Store -.python-version \ No newline at end of file +.python-version +.playwright-mcp/ diff --git a/AGENTS.md b/AGENTS.md index 15b7060..8fd1647 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,15 +120,15 @@ Response models live in `connector/models/` as Pydantic models. - `domain/` — thin view objects that hold a back-reference to the snapshot for cross-entity lookups - `model/` — raw Pydantic models for the API response (`collector.py`, etc.) -### Blueprints (standalone) +### Provisioning (standalone) -`blueprints/` is a standalone feature (`BlueprintEngine(app)`); `VideoIPathApp` neither imports nor exposes it. Design record: [`docs/architecture/blueprints/`](docs/architecture/blueprints/README.md). YAML parsing (PyYAML, a regular dependency) lives only in `loader.py`. -- `models.py` / `loader.py` / `resolution.py` — strict document models, safe YAML loading, variant merge + driver/parameter validation, JSON Schema -- `engine.py` — `BlueprintEngine`, read-only `plan()`, `BlueprintPlan.apply()` executor (phases, conflicts, partial results) +`provisioning/` is a standalone declarative configuration layer over the SDK (`ProvisioningEngine(app)`). Blueprints are reusable configuration templates within this layer. `VideoIPathApp` neither imports nor exposes it. Design record: [`docs/architecture/provisioning/`](docs/architecture/provisioning/README.md). YAML parsing (PyYAML, a regular dependency) lives only in `loader.py`. +- `models.py` / `loader.py` / `resolution.py` — strict document models, safe YAML loading, ordered variant overlays + typed inputs + driver/parameter validation, JSON Schema +- `engine.py` — `ProvisioningEngine`, read-only `plan()`, `ProvisioningPlan.apply()` executor (phases, conflicts, partial results); separate Inventory reachability (10s) and topology readiness (60s) budgets, with `require_reachable=False` for mock/static devices - `inventory.py` / `inspect.py` — the only modules that call `app.inventory` / `app.inspect` (managed-field diffs, fresh scoped reads, transaction/tag writes) - `naming.py` — `Text` / `Field` / `Join` naming blocks; `processors/` — `VertexProcessor` contract, per-engine `ProcessorRegistry`, built-in Matrox processor -- `schemas/blueprint-v1.schema.json` is generated; `tests/blueprints/test_resolution.py` fails when it is stale. Regenerate with: - `poetry run python -c "import json, pathlib; from videoipath_automation_tool.blueprints import Blueprint, ProcessorRegistry; pathlib.Path('src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json').write_text(json.dumps(Blueprint.json_schema(ProcessorRegistry()), indent=2, sort_keys=True) + '\n')"` +- `schemas/blueprint-v1.schema.json` is generated; `tests/provisioning/test_resolution.py` fails when it is stale. Regenerate with: + `poetry run python -c "import json, pathlib; from videoipath_automation_tool.provisioning import Blueprint, ProcessorRegistry; pathlib.Path('src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json').write_text(json.dumps(Blueprint.json_schema(ProcessorRegistry()), indent=2, sort_keys=True) + '\n')"` ### Driver versioning diff --git a/docs/architecture/blueprints/README.md b/docs/architecture/blueprints/README.md deleted file mode 100644 index 8f29f8c..0000000 --- a/docs/architecture/blueprints/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Blueprints — Architecture - -Design record for the blueprint feature in this package -(`src/videoipath_automation_tool/blueprints/`). - -A blueprint is a reusable description of one device type. `BlueprintEngine` combines -that description with the facts of one device, previews the changes, and writes -them through the existing Inventory and Inspect apps. `VideoIPathApp` does not -import or expose the feature. You construct the engine yourself. - -For a short introduction, start with the -[getting-started page](../../getting-started-guide/05_Blueprints.md). Runnable -scripts are in [docs/examples/07_blueprints](../../examples/07_blueprints/). -Offline tests live under `tests/blueprints/`. There is no live-server suite for -this feature yet. - -## Reading order - -1. **[reference.md](./reference.md)** — complete user reference: engine API, - `BlueprintDevice`, the YAML document, variants, loading, naming, apply phases, - errors, configuration, limits, and the public API index. -2. **[processors.md](./processors.md)** — the vertex processor contract and the - built-in Matrox ConvertIP processor. -3. **[concepts.md](./concepts.md)** — boundaries, module map, and the plan/apply flow. -4. **[decisions/](./decisions/)** — the choices that would be expensive to reverse. - Start with [the index](./decisions/README.md). - -The topology phase commits through Inspect's existing transaction. That write -model is recorded in -[Inspect ADR-004](../inspect-app/decisions/004-commit-write-model.md). - -## Decision log - -| Question | Decision | Status | -| -------- | -------- | ------ | -| Where does the feature sit in the package? | [ADR-001](./decisions/001-standalone-feature.md) | Accepted | -| How is a change previewed and written? | [ADR-002](./decisions/002-plan-then-apply.md) | Accepted | -| What does a blueprint update touch? | [ADR-003](./decisions/003-managed-fields.md) | Accepted | -| Who interprets discovered vertices? | [ADR-004](./decisions/004-processors-propose.md) | Accepted | -| Who may call Inventory and Inspect? | [ADR-005](./decisions/005-app-gateways.md) | Accepted | -| How do blueprint variants combine? | [ADR-006](./decisions/006-variant-merge.md) | Accepted | diff --git a/docs/architecture/provisioning/README.md b/docs/architecture/provisioning/README.md new file mode 100644 index 0000000..a6abfd4 --- /dev/null +++ b/docs/architecture/provisioning/README.md @@ -0,0 +1,78 @@ +# Provisioning — Architecture + +Design record for the declarative provisioning layer in this package +(`src/videoipath_automation_tool/provisioning/`). + +Provisioning automates device configuration in an Infrastructure as Code style +on top of the existing VideoIPath SDK. A blueprint is one part of this layer: +a reusable configuration template for a device type. `ProvisioningEngine` +combines that template with device facts, previews the changes, and writes them +through the existing Inventory and Inspect apps. `VideoIPathApp` does not import +or expose the feature. You construct the engine yourself. + +For a short introduction, start with the +[getting-started page](../../getting-started-guide/05_Provisioning.md). Runnable +scripts are in [docs/examples/07_provisioning](../../examples/07_provisioning/). +Offline tests live under `tests/provisioning/`. Live-server workflows live under +`tests/e2e/provisioning/`; see [E2E coverage](#e2e-coverage) below. + +## Reading order + +1. **[reference.md](./reference.md)** — complete user reference: engine API, + `ProvisioningDevice`, the YAML document, variants, loading, naming, apply phases, + errors, configuration, limits, and the public API index. +2. **[processors.md](./processors.md)** — the vertex processor contract and the + built-in Matrox ConvertIP processor. +3. **[concepts.md](./concepts.md)** — boundaries, module map, and the plan/apply flow. +4. **[decisions/](./decisions/)** — the choices that would be expensive to reverse. + Start with [the index](./decisions/README.md). + +The topology phase commits through Inspect's existing transaction. That write +model is recorded in +[Inspect ADR-004](../inspect-app/decisions/004-commit-write-model.md). + +## E2E coverage + +Run against a configured local test instance: + +```bash +poetry run test-e2e tests/e2e/provisioning +poetry run test-e2e tests/e2e/provisioning/test_lifecycle.py +poetry run test-e2e tests/e2e/provisioning/test_lifecycle.py::test_preview_apply_and_idempotent_reapply +``` + +The entry point loads the gitignored project `.env`, including legacy +`VIDEOIPATH_*` connection variables. Explicit paths select only those tests. +The suite requires VideoIPath 2025 or newer, the server's +`com.nevion.mock-0.1.0` driver, and permissions to configure Inventory, Inspect, +and catalog tags. It needs no physical devices. It is excluded from the default +offline/CI test run. + +The scenarios cover YAML inputs and variants, preview and dry run, onboarding, +Inventory/topology scope isolation, managed-field preservation, endpoint naming, +module-local tags, directed connections, deferred peers, rediscovery and sync +policies, stale plans, and recovery from an Inventory readiness timeout. Each +scenario owns its devices and checks persisted values through fresh server reads. +Mutation spies forward calls to the real SDK and verify that dry runs and +idempotent reapplications issue no writes. + +Devices use unique `E2E-` labels and the resolved `vipat-e2e` catalog tag. They +remain on the server for manual inspection after success or failure. As with +the rest of the E2E suite, the next E2E session removes **all** prior `E2E-` +resources, including resources from other E2E scenarios, before creating new +ones. Run live sessions serially against an instance. Discovery polling is bounded; +transport errors and unexpected failures are not retried. + +External connection behavior is recorded in [ADR-009](./decisions/009-external-connections.md). + +## Decision log + +| Question | Decision | Status | +| -------- | -------- | ------ | +| Where does the feature sit in the package? | [ADR-001](./decisions/001-standalone-feature.md) | Accepted | +| How is a change previewed and written? | [ADR-002](./decisions/002-plan-then-apply.md) | Accepted | +| What does a blueprint update touch? | [ADR-003](./decisions/003-managed-fields.md) | Accepted | +| Who interprets discovered vertices? | [ADR-004](./decisions/004-processors-propose.md) | Accepted | +| Who may call Inventory and Inspect? | [ADR-005](./decisions/005-app-gateways.md) | Accepted | +| How do blueprint inputs and variants resolve? | [ADR-007](./decisions/007-typed-inputs.md) | Accepted | +| When is a device ready for rollout? | [ADR-008](./decisions/008-staged-readiness.md) | Accepted | diff --git a/docs/architecture/blueprints/concepts.md b/docs/architecture/provisioning/concepts.md similarity index 64% rename from docs/architecture/blueprints/concepts.md rename to docs/architecture/provisioning/concepts.md index 716e6e7..6e75e1e 100644 --- a/docs/architecture/blueprints/concepts.md +++ b/docs/architecture/provisioning/concepts.md @@ -1,22 +1,28 @@ -# Blueprints — Concepts +# Provisioning — Concepts -Design record for the shipped blueprint package. Field-level usage is in +Design record for the provisioning package. Field-level usage is in [reference.md](./reference.md) and [processors.md](./processors.md). ## 1. What the feature is +The SDK apps expose the operations needed to configure VideoIPath yourself. +Provisioning adds a declarative layer that combines reusable configuration, +device facts, naming, and processors into a reviewable plan, then applies it +through those apps. + Inventory onboards a device. Inspect places it in the topology and edits -vertices. A blueprint describes how one *kind* of device should be configured -in both places, so a caller can apply that description to many real devices. +vertices. A blueprint is the reusable configuration template within provisioning: +it describes how one *kind* of device should be configured in both places, so a +caller can apply that description to many real devices. -The caller supplies the instance facts (`BlueprintDevice`): source-system id, +The caller supplies the instance facts (`ProvisioningDevice`): source-system id, label, addresses, credentials, an optional Inventory id, and an optional -Inspect target. The blueprint supplies the type: driver settings, topology -appearance, vertex interpretation, and naming. The engine does not know NetBox -or any other source system. Translation into `BlueprintDevice` stays outside +Inspect target, and concrete external connections. The blueprint supplies the type: driver settings, topology +appearance, vertex interpretation, port mapping, and naming. The engine does not know NetBox +or any other source system. Translation into `ProvisioningDevice` stays outside the package. -`plan()` reads current server state and returns a reviewable `BlueprintPlan`. +`plan()` reads current server state and returns a reviewable `ProvisioningPlan`. `plan.apply()` writes. The two calls share one executor. `engine.apply(...)` is `plan` followed by `apply`. @@ -49,7 +55,7 @@ flowchart LR inspectgw --> apps ``` -`BlueprintEngine` is constructed with a `BlueprintApp` protocol (`inventory` +`ProvisioningEngine` is constructed with a `ProvisioningApp` protocol (`inventory` and `inspect` properties). `VideoIPathApp` satisfies that protocol without being imported here. Construction does not touch the app or the network. @@ -69,12 +75,13 @@ keys, or custom tags. Limits are 1 MiB, 32 levels, and 100,000 nodes. | `naming.py` | `Text` / `Field` / `Join` expressions and layering. | | `processors/` | `VertexProcessor`, detached records, per-engine `ProcessorRegistry`, Matrox built-in. | | `inventory.py` | `InventoryGateway`: one Inventory record, managed-field diff, create/update. | +| `connections.py` | Pure generic port resolution, directional edge selection and managed-field diffs. | | `inspect.py` | `InspectGateway`: fresh scoped read, processor run, topology commit, module tags. | | `engine.py` | Configuration, `plan()`, captured inputs, phased `apply()`. | -| `errors.py` | `BlueprintError` hierarchy. Validation issues carry a path, not raw values. | -| `schemas/blueprint-v1.schema.json` | Generated document schema. `tests/blueprints/test_resolution.py` fails when the file is stale. | +| `errors.py` | `ProvisioningError` hierarchy. Validation issues carry a path, not raw values. | +| `schemas/blueprint-v1.schema.json` | Generated document schema. `tests/provisioning/test_resolution.py` fails when the file is stale. | -Public types are re-exported from `blueprints/__init__.py`. Engine internals +Public types are re-exported from `provisioning/__init__.py`. Engine internals (`_Planner`, `_Executor`, `_Captured`, the gateways' work models) are not. ## 4. From document to writes @@ -82,9 +89,9 @@ Public types are re-exported from `blueprints/__init__.py`. Engine internals ```mermaid flowchart TD yaml[Blueprint YAML or dict] - device[BlueprintDevice] + device[ProvisioningDevice] resolved[ResolvedBlueprint] - plan[BlueprintPlan] + plan[ProvisioningPlan] apply[apply] inv[Inventory create or update] disc[Discovery wait] @@ -101,26 +108,29 @@ flowchart TD apply --> inv --> disc --> sync --> topo --> tags --> verify ``` -Resolution ([ADR-006](./decisions/006-variant-merge.md)) selects `inventory` and -`topology` independently. A missing section is skipped. The selected named -entry patches `default`, then per-instance `inventory_overrides` patch the -inventory section. Driver `custom_settings` are checked against the driver -model for `SELECTED_SCHEMA_VERSION`. Processor `params` are checked against -the registered `params_model`. The source document is not mutated. Every merged -leaf keeps a provenance string (`default`, `variant:`, `instance`) so a -plan can say where a value came from. +Resolution ([ADR-007](./decisions/007-typed-inputs.md)) copies `defaults`, applies +caller-selected variant overlays in order, binds declared inputs in the effective +scope, then applies per-instance `inventory_overrides`. Variants may span both +Inventory and Topology. Unknown or repeated names fail. Input references retain +their types, replace whole nodes during merging, and are never evaluated a second +time. Inputs without defaults are needed only when the selected scope uses them. + +Driver `custom_settings` are checked against the model for +`SELECTED_SCHEMA_VERSION`; processor `params` against the registered `params_model`. +Every merged leaf retains provenance (`defaults`, `variant:`, `instance`, +with `|input:` for input values). The source document is never mutated. Naming is layered after that, entry by entry, later sources winning: library default, engine `naming`, resolved blueprint naming (a topology entry replaces the same inventory entry), then the `naming` argument on the call. An explicit -`None` turns that entry off. Names are rendered from facts on `BlueprintDevice`, -`module_position`, `EndpointIdentity`, and scalar `attributes`. They are not +`None` turns that entry off. Names are rendered from facts on `ProvisioningDevice`, +`module_position`, `EndpointIdentity`, scalar `attributes`, and bound `inputs`. They are not read off the current server label, so a second apply cannot grow a second suffix. ## 5. Planning `plan()` deep-copies the device, the blueprint, and the processor registry, and -stores them on the plan together with the resolved document, the layered naming +stores them on the plan together with copied inputs, ordered variants, the resolved document, the layered naming scheme, and the effective `ApplyOptions`. Later `engine.configure(...)` or `register_processor(...)` does not change a plan that already exists. @@ -132,7 +142,7 @@ Inventory planning compares managed fields with the current record or no change. A supplied `inventory_id` that does not exist fails. The engine does not create a replacement record. -Topology planning needs a target. That is `BlueprintDevice.topology`, or a +Topology planning needs a target. That is `ProvisioningDevice.topology`, or a `DeviceTarget` built from `inventory_id` when topology was omitted. When the exact edits cannot be known yet, the topology phases are marked `deferred` and `fully_resolved` is `False`. That happens when: @@ -162,14 +172,23 @@ replay in another process. ## 6. Applying -[ADR-002](./decisions/002-plan-then-apply.md) fixes the phase order: +[ADR-002](./decisions/002-plan-then-apply.md) defines phased execution; +[ADR-008](./decisions/008-staged-readiness.md) refines readiness into two budgets: 1. `inventory` — create or update the one record. A new id is stored on the result immediately. -2. `discovery` — poll while the scoped topology is not ready yet (`TopologyNotReadyError`), bounded by `discovery_timeout` (default 30 seconds, poll every 1 second). Any other error fails at once. -3. `topology_sync` — add the device or synchronize it, according to `ApplyOptions.sync`. -4. `topology` — device and vertex edits in one `InspectTransaction`, then commit. -5. `module_tags` — `assignTag` / `unassignTag` actions, one tag at a time. These are not part of the topology commit. -6. `verification` — read back what this apply wrote. +2. `inventory_readiness` — poll fresh Inventory status for `reachable=True`, bounded by `inventory_ready_timeout` (10 seconds). Missing status and unreachable devices remain pending. +3. `topology_sync` — add the device, confirm visibility, and synchronize it according to `ApplyOptions.sync`. +4. `discovery` — wait for required local topology data (`TopologyNotReadyError`). It shares a separate `topology_ready_timeout` (60 seconds) with topology sync; other errors fail at once. +5. `topology` — device, vertex and resolved edge edits in one `InspectTransaction`, then commit. +6. `module_tags` — `assignTag` / `unassignTag` actions, one tag at a time. These are not part of the topology commit. +7. `verification` — read back what this apply wrote. + +Both budgets use the injected monotonic clock and `poll_interval` (1 second). +Inventory-only applies, resolved topology, virtual targets, and module-only edits +without their own Inventory binding skip reachability. Mock/static devices can +opt out with `require_reachable=False`. Plans and dry runs never poll. Incomplete +local topology produces deferred plan phases, enabling a retry after partial +rollout. Peer discovery retains its existing explicit-replan behavior. An Inventory update that may alter discovery does not continue into discovery or topology in the same apply. Those phases stay `deferred`, the result status @@ -185,17 +204,19 @@ pending sync would update or remove elements, apply fails instead of escalating. conflict strategy, so Inspect's default `ConflictStrategy.STRICT` applies. A reported `syncDevices` failure is an error and is not retried. Adding a device that is not discovered yet stays `TopologyNotReadyError` and is retried -until `discovery_timeout`. +until `topology_ready_timeout`. A module target never reaches this add/sync path. Unchanged phases are skipped. An apply that would change nothing writes nothing. +A dry run performs the same stale-plan, conflict, and pending-edit checks without +writing. Changes report `planned`; work dependent on writes stays `deferred`. +It does not consume the plan and leaves verification `not_applicable`. -`dry_run=True` runs the same stale-plan, conflict, and pending-edit checks, then -returns before any write. Deferred topology stays `deferred` on a dry run, -because those edits depend on writes the dry run does not perform. A -topology-affecting update also stays `deferred`, with `replan_required` set. -`status` is `planned` when writes would have run, and `no_change` otherwise. +Use `engine.plan(...)` and `plan.summary()` for a read-only preview. Phases +whose edits depend on earlier writes stay `deferred` in the plan. Calling +`plan.apply()` executes the reviewed plan and checks current state before the +relevant writes. Before an Inventory update, the gateway re-reads the record, compares fingerprints of the managed fields, and repeats the label and address conflict @@ -208,10 +229,10 @@ Inspect edits that overlap the plan's device, vertices, or module are rejected. The Inspect transaction then does its own baseline check. These checks are client-side. A short gap between the read and the write remains. -If a later phase fails, earlier phases stay applied. `BlueprintApplyError.result` +If a later phase fails, earlier phases stay applied. `ProvisioningApplyError.result` is `partial` when something was written, `failed` when nothing was, and `unknown` when a write raised and the client cannot prove the server rejected -it (`InspectCommitError`, `InspectCommitConflictError`, `BlueprintError`, and +it (`InspectCommitError`, `InspectCommitConflictError`, `ProvisioningError`, and known Inventory `ValueError` messages count as known rejections, including a failed re-read that happens before `update_device` sends the update). A failed synchronization lookup is also an error; it is not treated as "nothing pending". @@ -228,5 +249,5 @@ Endpoint label uniqueness is checked per device, including endpoints outside the changed set, and not across the system. `ApplyOptions(naming_collisions="allow")` is the escape hatch. Names are not truncated or given a numeric suffix. -Offline tests under `tests/blueprints/` drive the engine with fakes. Live +Offline tests under `tests/provisioning/` drive the engine with fakes. Live behavior of the writes still has to be checked on a test system. diff --git a/docs/architecture/blueprints/decisions/001-standalone-feature.md b/docs/architecture/provisioning/decisions/001-standalone-feature.md similarity index 65% rename from docs/architecture/blueprints/decisions/001-standalone-feature.md rename to docs/architecture/provisioning/decisions/001-standalone-feature.md index d599bc2..9efb17c 100644 --- a/docs/architecture/blueprints/decisions/001-standalone-feature.md +++ b/docs/architecture/provisioning/decisions/001-standalone-feature.md @@ -5,20 +5,20 @@ ## Context Inventory, topology, preferences, profile, and security are properties of -`VideoIPathApp`. Blueprints sit on top of Inventory and Inspect and are optional -for callers who only onboard devices by hand. +`VideoIPathApp`. Provisioning sits on top of Inventory and Inspect and is optional +for callers who configure devices directly through the SDK apps. ## Options -- Add `app.blueprints`, initialized lazily like the other apps. -- Keep a separate `BlueprintEngine` that receives an app. +- Add `app.provisioning`, initialized lazily like the other apps. +- Keep a separate `ProvisioningEngine` that receives an app. ## Decision The feature is a separate package object. `VideoIPathApp` neither imports nor -exposes it. Callers write `BlueprintEngine(app)`. +exposes it. Callers write `ProvisioningEngine(app)`. -The engine depends on a `BlueprintApp` protocol with `inventory` and `inspect` +The engine depends on a `ProvisioningApp` protocol with `inventory` and `inspect` properties. Any object with those properties can be passed in. Construction does not read the app or open a connection. @@ -28,10 +28,10 @@ There is no process-global processor map. ## Consequences -- The rest of the package can be imported and released without pulling blueprint +- The rest of the package can be imported and released without pulling provisioning call sites into `VideoIPathApp`. - Tests can pass a small fake that satisfies the protocol. -- Discoverability is worse than `app.blueprints`. The getting-started page and +- Discoverability is worse than `app.provisioning`. The getting-started page and this record are the entry points. - Sharing an app across engines does not serialize writes. Callers who apply concurrently against one server have to coordinate themselves. diff --git a/docs/architecture/blueprints/decisions/002-plan-then-apply.md b/docs/architecture/provisioning/decisions/002-plan-then-apply.md similarity index 85% rename from docs/architecture/blueprints/decisions/002-plan-then-apply.md rename to docs/architecture/provisioning/decisions/002-plan-then-apply.md index 35bdec6..7dd267c 100644 --- a/docs/architecture/blueprints/decisions/002-plan-then-apply.md +++ b/docs/architecture/provisioning/decisions/002-plan-then-apply.md @@ -4,6 +4,9 @@ > > Field ownership: [ADR-003](./003-managed-fields.md). Gateways: > [ADR-005](./005-app-gateways.md). +> +> The original phase order and readiness timing below are refined by +> [ADR-008](./008-staged-readiness.md). ## Context @@ -20,19 +23,18 @@ same response. ## Decision -`engine.plan(...)` is read-only and returns an immutable `BlueprintPlan`. +`engine.plan(...)` is read-only and returns an immutable `ProvisioningPlan`. `plan.apply()` takes no configuration. It re-checks the captured baselines and executes that plan. `engine.apply(...)` is plan plus apply. -Phases run in order: inventory, discovery, topology sync, topology commit, +The original phase order was inventory, discovery, topology sync, topology commit, module tags, verification. Absent or unchanged phases are skipped. Inventory and Inspect writes are separate. Nothing rolls back an earlier phase. When topology edits depend on earlier work, the plan marks those phases `deferred` and sets `fully_resolved` to `False`. During apply the engine waits for discovery, then computes the edits from the blueprint, parameters, naming, -and target captured on the plan. A dry run does not invent those edits. It -reports the phases as `deferred`. +and target captured on the plan. An update of address, alternate addresses, credentials, generic or custom settings, or `active` can change the topology the driver discovers. Apply does @@ -40,8 +42,8 @@ not poll for that. It stops after Inventory, sets `replan_required`, and returns `status="partial"`. The caller plans again after rediscovery. Creating a record still waits for discovery, because a new device has no previous topology. -`dry_run=True` runs the stale-plan, conflict, and pending-edit checks and -performs no write. +`plan()` provides the read-only preview. `apply()` runs the stale-plan, +conflict, and pending-edit checks before the relevant writes. ## Consequences diff --git a/docs/architecture/blueprints/decisions/003-managed-fields.md b/docs/architecture/provisioning/decisions/003-managed-fields.md similarity index 100% rename from docs/architecture/blueprints/decisions/003-managed-fields.md rename to docs/architecture/provisioning/decisions/003-managed-fields.md diff --git a/docs/architecture/blueprints/decisions/004-processors-propose.md b/docs/architecture/provisioning/decisions/004-processors-propose.md similarity index 85% rename from docs/architecture/blueprints/decisions/004-processors-propose.md rename to docs/architecture/provisioning/decisions/004-processors-propose.md index cfa6a37..188b4d2 100644 --- a/docs/architecture/blueprints/decisions/004-processors-propose.md +++ b/docs/architecture/provisioning/decisions/004-processors-propose.md @@ -3,6 +3,10 @@ > Status: **Accepted** > > Where the proposal is computed: [ADR-005](./005-app-gateways.md). +> +> The original retry policy below is superseded by +> [ADR-008](./008-staged-readiness.md): only `TopologyNotReadyError` is retried; +> `ProcessorInputError` fails immediately. ## Context @@ -37,9 +41,9 @@ ports need a codec vertex, `supports_static_igmp` an IP vertex). Explicit `topol processor device defaults. Naming overrides proposed labels. Explicit `vertices` overrides win over both. -`ProcessorInputError` means the layout is unsupported or not ready yet. While -topology work is deferred, the engine retries that error until -`discovery_timeout`. +The original policy treated `ProcessorInputError` as an unsupported or not-yet-ready +layout and retried it during deferred topology work until `discovery_timeout`. +See ADR-008 for the current distinction between readiness and permanent errors. Registration is per engine ([ADR-001](./001-standalone-feature.md)). Duplicate ids fail, including the built-in `matrox.convertip.default`. YAML names the diff --git a/docs/architecture/blueprints/decisions/005-app-gateways.md b/docs/architecture/provisioning/decisions/005-app-gateways.md similarity index 92% rename from docs/architecture/blueprints/decisions/005-app-gateways.md rename to docs/architecture/provisioning/decisions/005-app-gateways.md index 5033c48..91d5979 100644 --- a/docs/architecture/blueprints/decisions/005-app-gateways.md +++ b/docs/architecture/provisioning/decisions/005-app-gateways.md @@ -26,16 +26,16 @@ record, resolves an SNMP configuration by exact label or id, creates with `find_device_ids_by_addresses`: one read for all addresses, compared after normalization on both sides (IP literals in compressed form, other identifiers case-insensitively). An empty item list is no match. A response with no item -list raises; the gateway turns that into `BlueprintError`. +list raises; the gateway turns that into `ProvisioningError`. `InspectGateway` is the only object that calls `app.inspect`. Scope data is built from fresh collector and lookup reads (`get_device_detail`, `lookup_inspect_device`, `lookup_vertices`), not from the live snapshot's domain objects. If the collector already has the device, a failed edit-form -lookup is `BlueprintError`. The virtual-device fallback, where there is no +lookup is `ProvisioningError`. The virtual-device fallback, where there is no topology node, still treats a missing form as absent. Pending edits are visible only through `staged_edit_keys()`, which is used to reject overlap, not to seed -the plan. A reported `syncDevices` failure is `BlueprintError` and is not +the plan. A reported `syncDevices` failure is `ProvisioningError` and is not retried. `addDevices` failure stays `TopologyNotReadyError`. Topology writes go through one `InspectTransaction`: `update_device` and diff --git a/docs/architecture/blueprints/decisions/006-variant-merge.md b/docs/architecture/provisioning/decisions/006-variant-merge.md similarity index 87% rename from docs/architecture/blueprints/decisions/006-variant-merge.md rename to docs/architecture/provisioning/decisions/006-variant-merge.md index 0978572..6292296 100644 --- a/docs/architecture/blueprints/decisions/006-variant-merge.md +++ b/docs/architecture/provisioning/decisions/006-variant-merge.md @@ -1,6 +1,6 @@ # ADR-006: Variants are a fixed patch over `default` -> Status: **Accepted** +> Status: **Superseded by [ADR-007](./007-typed-inputs.md)** > > What a merged field means at write time: [ADR-003](./003-managed-fields.md). @@ -34,16 +34,15 @@ A named entry patches that section's `default`: parameters included - a different `driver_id` drops the inherited `custom_settings` -`inventory_overrides` on `BlueprintDevice` are a third patch, applied only to +`inventory_overrides` on `ProvisioningDevice` are a third patch, applied only to the inventory section, and recorded as provenance `instance`. The loader rejects YAML aliases, anchors, and merge keys, so a document cannot smuggle a second composition language past the merge rules. Documents that need generation are built in Python and passed to `Blueprint.from_dict`. -`schema_version` is part of the document model. The published JSON Schema is -`schemas/blueprint-v1.schema.json`, regenerated from `Blueprint.json_schema`. -The unit test fails when the committed file drifts. +`schema_version` is part of this superseded document model. The published schema +is defined by [ADR-007](./007-typed-inputs.md). ## Consequences diff --git a/docs/architecture/provisioning/decisions/007-typed-inputs.md b/docs/architecture/provisioning/decisions/007-typed-inputs.md new file mode 100644 index 0000000..cb295c6 --- /dev/null +++ b/docs/architecture/provisioning/decisions/007-typed-inputs.md @@ -0,0 +1,51 @@ +# ADR-007: Typed inputs and ordered overlays + +> Status: **Accepted** +> +> Supersedes [ADR-006](./006-variant-merge.md). Managed-field semantics remain +> defined by [ADR-003](./003-managed-fields.md). + +## Context + +Reusable device configuration needs instance values from arbitrary callers. +Separate Inventory and Topology variant axes limit composition and mix base +configuration with named patches. Source-system models do not belong in YAML. + +## Decision + +The document has `schema_version: 1`, typed `inputs`, Inventory/Topology +`defaults`, and optional `variants`. Each variant can patch either section or +both. Callers explicitly select an ordered sequence; later variants win. + +Resolution copies defaults, merges overlays, inserts typed input values, +applies instance Inventory overrides, then validates the effective configuration. +Mappings merge; lists, tag specifications and individual naming expressions +replace. Driver changes discard custom settings; processor changes replace +processor parameters. Input and literal operators replace whole nodes. + +Input declarations use a strict, recursive JSON Schema vocabulary for scalar, +array and object values. Objects are closed by default. Supplied values replace +input defaults without coercion. Missing inputs fail when the effective scope +uses them. `{$input: name}` inserts a value; `{$literal: ...}` escapes operators. +Inserted values are never interpreted again. Naming can use input scalar leaves. + +The existing runtime configuration models also define document template fields +and editor schemas. Runtime driver and registered processor validation follows +binding. Loader errors retain YAML positions; resolved leaves retain YAML and +input provenance. Source ids, credentials and source-system adapters stay with +the caller. No source system selects variants implicitly. + +The generated schema is `blueprint-v1.schema.json`. Offline validation checks +defaults and each variant separately, or one explicit combination; planning +checks the chosen combination and scope before server I/O. + +## Consequences + +The same blueprint works with hand-written facts, NetBox, CSV or any other +adapter. Existing configuration and execution capabilities remain available. +Callers can combine independent policies and device layouts without creating a +variant for every pair. + +Plans capture copied input values and the selected order, so later caller edits +do not change execution. YAML remains self-contained and uses the safe loader; +there is no import, expression-evaluation or fleet-execution language. diff --git a/docs/architecture/provisioning/decisions/008-staged-readiness.md b/docs/architecture/provisioning/decisions/008-staged-readiness.md new file mode 100644 index 0000000..14b3f56 --- /dev/null +++ b/docs/architecture/provisioning/decisions/008-staged-readiness.md @@ -0,0 +1,72 @@ +# ADR-008: Independent Inventory and topology readiness deadlines + +> Status: **Accepted** +> +> Refines the phase order and readiness timing in [ADR-002](./002-plan-then-apply.md) +> and supersedes the retry policy in [ADR-004](./004-processors-propose.md). + +## Context + +An accepted Inventory write does not mean VideoIPath can connect to the device. +Topology membership, synchronization, ports, and vertices can appear later. +A shared timeout obscures which stage stalled and lets Inventory waiting consume +the time needed for topology discovery. Mock and static devices may never report +reachability even though their topology is usable. + +## Decision + +Apply phases are `inventory`, `inventory_readiness`, `topology_sync`, `discovery`, +`topology`, `module_tags`, and `verification`. + +Before deferred topology rollout for an Inventory-backed device, poll fresh +Inventory status until `reachable=True`. Each attempt calls +`InventoryApp.refresh_device_status()` once; the executor does not use the +SDK's internally retried `get_device(config_only=False)`. Only +`InventoryStatusUnavailableError` and `reachable=False` remain pending. +Transport failures and malformed status fail immediately. + +`ApplyOptions.inventory_ready_timeout` defaults to 10 seconds. After that gate +succeeds, a new `topology_ready_timeout` budget of 60 seconds covers topology +addition, synchronization, and the required ports and vertices. Both timeouts +and `poll_interval` (default 1 second) must be positive and finite. Deadlines use +the injected monotonic clock, include request time, and never restart on +progress. The first check is immediate; sleeps are capped to the remaining +budget. No new readiness requests or rollout actions begin after expiry. +In-flight HTTP requests retain their connector-level timeouts. + +`require_reachable=False` explicitly bypasses the Inventory gate for mock or +static devices. Virtual targets and module-only topology targets without their +own Inventory binding skip it. Otherwise, check the created or bound Inventory +record, falling back to a physical topology target's Inventory record. A module +target never causes addition or synchronization of its entire parent device. + +Inventory-only applies finish after Inventory. Fully resolved topology plans do +not acquire a reachability requirement. Topology-changing Inventory updates +still stop with `partial` and `replan_required=True` before rollout. + +Topology polling reads fresh membership, synchronization state, and scoped data. +Confirm membership and finish permitted synchronization before configuration +writes. An accepted addition is not resubmitted while membership or ports are +pending. An accepted synchronization with the same pending changes is polled; +newly discovered changes may require another synchronization under the captured +policy. Permanent processor, capability, and synchronization errors fail at once. +Only local `TopologyNotReadyError` signals missing data that can be retried. +Unknown write outcomes are never automatically retried. + +Planning converts local `TopologyNotReadyError` into deferred topology work, +including when an earlier rollout already added the device. Planning and dry +runs never sleep, and dry runs report deferred readiness without rollout actions. +Missing peer connections continue to require a new plan. + +## Consequences + +- `discovery_timeout` is removed without an alias. Callers configure the two + deadlines and, where needed, the explicit reachability bypass. +- Inventory timeouts raise `InventoryNotReadyError`; topology timeouts raise + `TopologyNotReadyError`. Both identify the device, stage, configured timeout, + and last observed condition in the partial result's failed phase. +- Known IDs and successful operations survive failure. A caller can bind the + returned Inventory ID and plan again without creating a duplicate record. +- Timeouts bound readiness polling, not an entire apply or a request already in + flight. Configuration writes and verification follow successful readiness. +- Blueprint YAML and its generated schema are unchanged. diff --git a/docs/architecture/provisioning/decisions/009-external-connections.md b/docs/architecture/provisioning/decisions/009-external-connections.md new file mode 100644 index 0000000..a6b4058 --- /dev/null +++ b/docs/architecture/provisioning/decisions/009-external-connections.md @@ -0,0 +1,47 @@ +# ADR-009: Concrete external connections belong to the device instance + +> Status: **Accepted** +> +> Extends [ADR-003](./003-managed-fields.md) and [ADR-005](./005-app-gateways.md). + +## Context + +Provisioning a device includes configuring its external connections. Reusable +device templates must remain independent of instance-specific cabling and the +system supplying it. IP interfaces and directed media ports need the same flow. + +## Decision + +`TopologyConfig.port_mapping` maps reusable local names to ordered port selectors +in the blueprint document. Existing overlay and typed-input semantics apply. The separate +`ip_vertex_mapping` and processor `interfaces` retain their existing behavior; +generic mappings are exposed as `port_bindings`. + +`ProvisioningDevice.connections` declares each concrete connection through +`ProvisioningConnection`, `PeerEndpoint` and `EdgePatch`. The local port is a +mapping key or direct selector. Peers use exact VideoIPath device/module ids +and port selectors. Source-system adapters remain outside provisioning. + +Direction is derived from actual Out/In vertices, with explicit direction +overrides. Vertex ids are never synthesized from naming conventions. Edges are +identified by their directed vertex pair, created when absent, and updated only +in explicitly managed fields. Other edges are preserved; empty input never prunes. + +The Inspect gateway reads peers and edge forms freshly, checks captured baselines +and staged-edit overlap, and commits device, vertex and edge intents together in +the existing topology transaction. Pure resolution/diff logic lives in +`connections.py`; processors receive no gateway or external write scope. + +Missing peer discovery is reported per connection as `deferred`. Remaining local +work and resolved connections can run. The result is `partial` with +`replan_required=True`, even if no write was necessary. A new plan is needed when +peers appear; applying an old plan does not adopt new peer topology. Invalid +selectors, ambiguous directions and failed reads are errors, not pending work. + +## Consequences + +- Blueprint and instance facts fully describe the requested configuration. +- One connection can produce one directed edge or both directions. +- Missing peers do not prevent otherwise valid local onboarding. +- Plans and results expose bindings, concrete edge ids and unresolved reasons. +- No peer provisioning, implicit synchronization, pruning or rollback is added. diff --git a/docs/architecture/blueprints/decisions/README.md b/docs/architecture/provisioning/decisions/README.md similarity index 66% rename from docs/architecture/blueprints/decisions/README.md rename to docs/architecture/provisioning/decisions/README.md index 71d91de..cbdc6ed 100644 --- a/docs/architecture/blueprints/decisions/README.md +++ b/docs/architecture/provisioning/decisions/README.md @@ -1,4 +1,4 @@ -# Architecture Decision Records — Blueprints +# Architecture Decision Records — Provisioning Each ADR records one decision that would be expensive to reverse: the context, the choice, and the consequences. Supersede a decision with a new ADR rather @@ -13,4 +13,7 @@ than rewriting history. | [003](./003-managed-fields.md) | Updates touch only managed fields | Accepted | | [004](./004-processors-propose.md) | Processors propose, the engine writes | Accepted | | [005](./005-app-gateways.md) | Inventory and Inspect are reached through two gateways | Accepted | -| [006](./006-variant-merge.md) | Variants are a fixed patch over `default` | Accepted | +| [006](./006-variant-merge.md) | Variants are a fixed patch over `default` | Superseded by ADR-007 | +| [007](./007-typed-inputs.md) | Typed inputs and ordered overlays | Accepted | +| [008](./008-staged-readiness.md) | Independent Inventory and topology readiness deadlines | Accepted | +| [009](./009-external-connections.md) | Concrete external connections belong to the device instance | Accepted | diff --git a/docs/architecture/blueprints/processors.md b/docs/architecture/provisioning/processors.md similarity index 90% rename from docs/architecture/blueprints/processors.md rename to docs/architecture/provisioning/processors.md index 9160d7b..d0c7b57 100644 --- a/docs/architecture/blueprints/processors.md +++ b/docs/architecture/provisioning/processors.md @@ -1,16 +1,16 @@ -# Blueprints — Vertex processors +# Provisioning — Vertex processors A processor reads the vertices one driver discovered and proposes configuration. The engine validates the proposal, applies naming, compares it with the server, and writes ([ADR-004](./decisions/004-processors-propose.md)). The rest of the -blueprint reference is in [reference.md](./reference.md). +provisioning reference is in [reference.md](./reference.md). ## 1. Writing a processor ```python from pydantic import BaseModel, ConfigDict -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( EndpointIdentity, ProcessingContext, ProcessorResult, VertexEdit, VertexPatch, VertexProcessor, ) @@ -40,16 +40,17 @@ engine.register_processor("example-org.simple-video", SimpleVideoProcessor) ``` ```yaml -topology: - default: +defaults: + topology: vertex_processor: processor_type: example-org.simple-video params: {factory_label: Video Sender} ``` A processor subclasses `VertexProcessor[ParamsT]`, declares `params_model` (a -Pydantic model class), and implements `process(context, params)`. The engine -validates the YAML `params` against `params_model` before calling `process`, and +Pydantic model class), and implements `process(context, params)`. Typed inputs may supply `params` or individual parameter values using +`{$input: name}`. The engine +validates the resolved `params` against `params_model` before calling `process`, and creates a new processor instance for every run, so an instance may keep run-local state. @@ -59,10 +60,11 @@ run-local state. | Field | Content | |---|---| -| `source` | `SourceFacts`: `key`, `label`, `description`, `module_position`, `attributes` from `BlueprintDevice`. No credentials. | +| `source` | `SourceFacts`: `key`, `label`, `description`, `module_position`, `attributes` from `ProvisioningDevice`. No credentials. | | `scope`, `device_id`, `module_id` | `"device"` or `"module"`, and the target ids. | | `device` | `DeviceRecord`: label, description, factory label, icon, SDP strategy, site, coordinates, tags. | | `modules`, `ports`, `vertices` | `ModuleRecord`, `PortRecord`, `VertexRecord` tuples, limited to the target scope. A module target sees only its own module. | +| `port_bindings` | Generic `port_mapping` bindings, separate from the legacy IP-only `interfaces`. | | `interfaces` | The resolved `ip_vertex_mapping` as `InterfaceBinding`s (`key`, `candidate`, `port_id`, `in_vertex_id`, `out_vertex_id`). | | `owner` | `DriverContext` of the topology owner's Inventory record: `inventory_id`, `driver_id`, `address`, `custom_settings`. | | `inventory` | The source entity's own `DriverContext` when it differs from the owner (a module with its own record). | @@ -101,12 +103,12 @@ both. Raise `TopologyNotReadyError` when discovery is still incomplete (for example, vertices without an edit form, or no codec vertices yet). While topology work is -deferred, the engine retries only this error, until `discovery_timeout`. Raise +deferred, the engine retries only this error, until `topology_ready_timeout`. Raise `ProcessorInputError` when the layout is unsupported; the apply fails at once. ## 4. Registration -Registration belongs to one engine (`BlueprintEngine(app, processors={...})` or +Registration belongs to one engine (`ProvisioningEngine(app, processors={...})` or `engine.register_processor(id, cls)`). Ids use letters, digits, `.`, `_`, and `-`. Registering an id that already exists, including a built-in id, fails. YAML names the registered id; it does not name a Python import path. @@ -146,7 +148,7 @@ does not recognize. | `video_receiver_tags`, `video_sender_tags`, `audio_receiver_tags`, `audio_sender_tags` | omitted | Category tags. Leave the parameter out to leave tags alone, pass `[]` to clear them, or pass `{add, remove}` to add and remove. | | `redundant_streams` | omitted | `true` sets `SIPSAuto`, `false` sets `NONE`, and omitting the parameter leaves the value alone. `true` requires exactly two streaming IP vertices. Use `ip_vertex_mapping` when the device also has management ports. The processor does not check that paths actually run through that pair. | | `allow_null_audio_label` | `false` | When `true`, the single codec vertex labelled exactly `null`, which cannot be classified, is accepted as audio. The processor emits a warning diagnostic. | -| `configure_control` | `false` | Not available yet. Setting it raises `BlueprintCapabilityError`. The Inspect control mapping is still unverified, so control settings on the device stay as they are. | +| `configure_control` | `false` | Not available yet. Setting it raises `ProvisioningCapabilityError`. The Inspect control mapping is still unverified, so control settings on the device stay as they are. | The proposal enables the endpoints and assigns an `EndpointIdentity` with engine `0` and program `1`. The factory label becomes the description, joined with the diff --git a/docs/architecture/blueprints/reference.md b/docs/architecture/provisioning/reference.md similarity index 53% rename from docs/architecture/blueprints/reference.md rename to docs/architecture/provisioning/reference.md index f1a8d52..9b2d512 100644 --- a/docs/architecture/blueprints/reference.md +++ b/docs/architecture/provisioning/reference.md @@ -1,7 +1,7 @@ -# Blueprints — Reference +# Provisioning — Reference -Complete user reference for `videoipath_automation_tool.blueprints`. For a short -introduction, read the [getting-started page](../../getting-started-guide/05_Blueprints.md) +Complete user reference for `videoipath_automation_tool.provisioning`. For a short +introduction, read the [getting-started page](../../getting-started-guide/05_Provisioning.md) first. Why the feature works this way is in [concepts.md](./concepts.md) and the [decisions](./decisions/README.md). Vertex processors, including the built-in Matrox processor, have their own page: [processors.md](./processors.md). @@ -9,7 +9,7 @@ Matrox processor, have their own page: [processors.md](./processors.md). ## Contents 1. [Engine API](#1-engine-api) -2. [Describing a device: `BlueprintDevice`](#2-describing-a-device-blueprintdevice) +2. [Describing a device: `ProvisioningDevice`](#2-describing-a-device-provisioningdevice) 3. [Blueprint document](#3-blueprint-document) 4. [Variants](#4-variants) 5. [Loading and validation](#5-loading-and-validation) @@ -20,19 +20,20 @@ Matrox processor, have their own page: [processors.md](./processors.md). 10. [Source-system adapters](#10-source-system-adapters) 11. [Limits and verification status](#11-limits-and-verification-status) 12. [Public API index](#12-public-api-index) +13. [Migrating from the former API](#13-migrating-from-the-former-api) ## 1. Engine API -`BlueprintEngine(app, *, processors=None, options=None, naming=None, logger=None)` -takes any object with `inventory` and `inspect` properties (the `BlueprintApp` -protocol). `VideoIPathApp` satisfies it. The app has no blueprint attribute, so +`ProvisioningEngine(app, *, processors=None, options=None, naming=None, logger=None)` +takes any object with `inventory` and `inspect` properties (the `ProvisioningApp` +protocol). `VideoIPathApp` satisfies it. The app has no provisioning attribute, so you construct the engine yourself. | Member | Purpose | |---|---| -| `plan(device, blueprint, *, scope="all", inventory_variant="default", topology_variant="default", naming=None, options=None)` | Read-only. Loads and resolves the blueprint, reads current state, returns a `BlueprintPlan`. | +| `plan(device, blueprint, *, scope="all", inputs=None, variants=(), naming=None, options=None)` | Read-only. Loads and resolves the blueprint, reads current state, returns a `ProvisioningPlan`. | | `apply(device, blueprint, *, ..., dry_run=False)` | Same arguments as `plan`, plus `dry_run`. Identical to `plan(...).apply(dry_run=...)`. | -| `validate(blueprint)` | Offline. Resolves every variant and reports unregistered processors. | +| `validate(blueprint, *, inputs=None, variants=None)` | Offline. Checks the base and every individual variant, or one explicit combination, including registered processor parameters. | | `json_schema()` | Document JSON Schema for this engine: `processor_type` limited to the registered processors, with each one's parameters. | | `register_processor(processor_id, processor_cls)` | Registers a processor for this engine only. See [processors.md](./processors.md). | | `configure(*, options=..., naming=...)` | Replaces engine defaults. See [section 9](#9-engine-configuration). | @@ -42,17 +43,18 @@ you construct the engine yourself. a file path string, a `pathlib.Path` (or another `os.PathLike[str]`), or a `Blueprint` instance. -**`BlueprintPlan`** +**`ProvisioningPlan`** | Member | Meaning | |---|---| -| `apply(*, dry_run=False)` | Re-checks the plan's assumptions and executes it. A dry run performs every check but no write. | +| `apply(*, dry_run=False)` | Re-checks the plan's assumptions and executes it. A dry run runs the checks without writes. | | `summary()` | Human-readable, redacted before/after listing, bindings, diagnostics, and unresolved work. | | `phases`, `phase(name)` | `PlannedPhase` records (`planned`, `no_change`, `deferred`, `skipped`) with their `PlannedOperation`s and `FieldChange`s. | | `has_changes` | `True` when any phase is `planned` or `deferred`. | -| `fully_resolved` | `False` when topology phases are deferred until apply, or until a new plan after a topology-affecting Inventory update. | -| `interface_bindings`, `diagnostics`, `skipped_sections` | Resolved `ip_vertex_mapping`, warnings, and sections the scope skipped. | -| `source_key`, `scope`, `inventory_variant`, `topology_variant`, `inventory_id`, `topology_target`, `driver_id`, `driver_schema_version`, `processor_type`, `blueprint_digest` | Identity of what was planned. | +| `fully_resolved` | `False` when local topology work is deferred, an Inventory update requires replanning, or peer connections remain unresolved. | +| `interface_bindings`, `port_bindings`, `connections` | Legacy IP bindings, generic port bindings, and each connection’s endpoints, edge ids, status and unresolved reason. | +| `diagnostics`, `skipped_sections` | Warnings and sections the scope skipped. | +| `source_key`, `scope`, `variants`, `inventory_id`, `topology_target`, `driver_id`, `driver_schema_version`, `processor_type`, `blueprint_digest` | Identity of what was planned. | A plan is an in-memory object bound to the app it was built with. It also holds unredacted Inventory values for the write, so it is not a document you save and @@ -64,17 +66,17 @@ replay elsewhere. |---|---| | `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. `partial` with `replan_required` is returned, not raised. | | `ok` | `True` for `succeeded`, `no_change`, and `planned`. | -| `replan_required` | `True` when an Inventory update may change discovery. Apply stopped after Inventory. `status` is `partial`, or `planned` on a dry run. Plan again after rediscovery. | +| `replan_required` | `True` after an Inventory update requiring rediscovery, or when peer connections remain open. Real runs return `partial`; dry runs return `planned`. Create a new plan when discovery is ready. | | `inventory_id`, `topology_device_id`, `module_id`, `source_key` | Ids involved. Persist `inventory_id` in your source system. | -| `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. `deferred` also appears when a new plan is required, not only in a dry run. | -| `interface_bindings`, `diagnostics` | As on the plan, after deferred work was materialized. Empty when apply stopped for a replan. | +| `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. `deferred` reports work still waiting for discovery or a new plan. | +| `interface_bindings`, `port_bindings`, `connections`, `diagnostics` | Resolved bindings and connection outcomes after execution. Deferred connections retain the reason and available endpoint information. | | `materialized` | `True` when deferred topology work was computed during apply. | | `verification`, `verification_detail` | `confirmed`, `unconfirmed`, or `not_applicable`. | -| `dry_run` | Whether this was a dry run. | +| `dry_run` | Whether the execution performed checks without writes. | -## 2. Describing a device: `BlueprintDevice` +## 2. Describing a device: `ProvisioningDevice` -`BlueprintDevice` is one device, module, or virtual device, plus the facts only +`ProvisioningDevice` is one device, module, or virtual device, plus the facts only your source system knows. | Field | Meaning | @@ -87,9 +89,10 @@ your source system knows. | `topology` | `DeviceTarget(device_id)` or `ModuleTarget(device_id, module_id)`, using exact Inspect ids. Leave it out to target the Inventory id, including an id this same plan creates. | | `module_position` | Module context you supply for naming and mappings. It is a string, such as `"0"`, `"A1"`, or `"1/2"`. | | `inventory_overrides` | Per-device Inventory settings that are not secrets (`InventorySettings`): `custom_settings`, `generic_settings`, `snmp`, `metadata`, `active`. Applied after the selected variant. | +| `connections` | Concrete external connections: local port, peer device/module and port, direction and managed edge fields. Defaults to `[]`; does not remove existing edges. | | `attributes` | JSON facts you own. Scalar leaves are available to naming as `attributes.`, and processors see them on `context.source`. | -`BlueprintDevice.from_inventory(inventory_device, key=None)` binds an existing +`ProvisioningDevice.from_inventory(inventory_device, key=None)` binds an existing Inventory device by id, label, and description. These are the arrangements the engine supports: @@ -108,34 +111,95 @@ a display name. The engine does not use it as identity. ```yaml schema_version: 1 - -inventory: - default: +defaults: + inventory: driver_id: com.nevion.NMOS_multidevice-0.1.0 - generic_settings: {enable_https: false, http_auth: 0} - custom_settings: {port: 8080, disable_rx_sdp_with_null: true} - -topology: - default: - device: {icon_size: medium} + generic_settings: + enable_https: false + http_auth: 0 + custom_settings: + port: 8080 + disable_rx_sdp_with_null: true + topology: + device: + icon_size: medium ip_vertex_mapping: - stream-a: [P1] - stream-b: [P2] + stream-a: + - P1 + stream-b: + - P2 vertex_processor: processor_type: matrox.convertip.default params: redundant_streams: true - video_receiver_tags: [Format~~video-tag-a] - video_sender_tags: [Format~~video-tag-b] + video_receiver_tags: + - Format~~video-tag-a + video_sender_tags: + - Format~~video-tag-b +variants: receiver: - vertex_processor: - params: {mode: rx} + topology: + vertex_processor: + params: + mode: rx ``` -`schema_version` must be `1`. `inventory` and `topology` are each optional, and -the file needs at least one of them. A section you include must contain a -`default` entry. Variant names start with a letter or digit and may contain -letters, digits, `_`, `.`, and `-`. +`schema_version` must be the integer `1`. `defaults` contains `inventory`, +`topology`, or both. `inputs` and `variants` are optional. Each variant contains +Inventory and/or Topology configuration patches. Variant names start with a +letter or digit and may contain letters, digits, `_`, `.`, and `-`. + +### Typed inputs + +Inputs declare values the caller supplies independently of any source system: + +```yaml +inputs: + api_port: + type: integer + default: 8080 + location: + type: object + properties: + code: {type: string} + required: [code] + video_tags: + type: array + items: {type: string} + default: [] +defaults: + inventory: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + custom_settings: + port: {$input: api_port} +``` + +`type` supports `string`, `integer`, `number`, `boolean`, `array`, `object`, +and `null`. Definitions also accept `description`, `default`, and `enum`. +Arrays accept a recursive `items` definition. Objects accept recursive +`properties`, a `required` property-name list, and `additionalProperties` +(default `false`). Types are strict: a boolean is not an integer, and a numeric +string is not converted. Defaults and enum entries must conform to the declaration. + +A supplied input replaces its whole default; nested property defaults are not +inserted automatically. An input without a default is required when the effective +selected scope uses it. Unknown supplied inputs or declared-reference names fail. +Naming can use scalar leaves as `{inputs.location.code}`. Input names start with +a letter or underscore and may contain letters, digits, `_`, and `-`. + +`{$input: api_port}` inserts a typed value at a configuration value position, +including whole lists and objects. `{$literal: ...}` escapes a value containing +operator keys. Each operator mapping contains exactly one key. Inserted values +are never evaluated again as operators. Section keys, driver ids, and processor +declarations stay static; processor parameters can contain references. + +```python +plan = engine.plan( + device, blueprint, + inputs={"location": {"code": "site-a"}, "video_tags": ["video-tag-a"]}, + variants=["secure", "receiver"], +) +``` ### Inventory entry @@ -159,6 +223,7 @@ letters, digits, `_`, `.`, and `-`. | `vertex_processor` | `processor_type` and `params`. The registered processor validates `params`. See [processors.md](./processors.md). | | `vertices` | Explicit overrides. Point at the vertex with `vertex_id`, or with `factory_label` (may contain `{module.position}`) optionally narrowed by `kind` and `direction` (`In`, `Out`, `Internal`, `Undecided`), and supply `fields`. Those fields use the same contract as processor output. Each override must match exactly one vertex. | | `naming` | `device_label`, `device_description`, `endpoint_label`, `endpoint_description`. | +| `port_mapping` | Generic local connection names mapped to ordered lists of `PortSelector` objects. Supports typed inputs and variant overlays. Keys must differ from `ip_vertex_mapping`. | ### Vertex fields @@ -191,26 +256,135 @@ server. The engine does not store what it applied last time. If a variant should turn a stream off, replace a tag set, or clear an endpoint flag, put that new value in the variant. Background: [ADR-003](./decisions/003-managed-fields.md). -## 4. Variants +### External connections -`inventory_variant` and `topology_variant` are chosen independently. Both default -to `default`. An unknown name fails, and the error lists the names that exist. -There is no silent fallback to `default`. +Declare concrete connections on **`ProvisioningDevice.connections`**. The YAML +blueprint provides reusable local port names; peer identities and edge settings +are instance facts supplied by the caller. -A named entry is a patch on top of that section's `default` -([ADR-006](./decisions/006-variant-merge.md)): +```yaml +schema_version: 1 +defaults: + topology: + port_mapping: + uplink: + - factory_label: port-1 +``` + +```python +from videoipath_automation_tool.provisioning import ( + DeviceTarget, EdgePatch, PeerEndpoint, PortSelector, + ProvisioningConnection, ProvisioningDevice, ProvisioningEngine, +) + +device = ProvisioningDevice( + key="device-a", + label="device-a", + inventory_id="device1", + connections=[ + ProvisioningConnection( + local="uplink", + peer=PeerEndpoint( + target=DeviceTarget(device_id="device2"), + port=PortSelector(factory_label="port-1"), + ), + direction="auto", + fields=EdgePatch(weight=10, label="edge-a"), + ), + ], +) +engine = ProvisioningEngine(app) # an existing VideoIPathApp +plan = engine.plan(device, "external-connections.yml", scope="topology") +print(plan.summary()) +result = plan.apply() +for connection in result.connections: + print(connection.index, connection.status, connection.edge_ids, connection.reason) +``` -- Mappings merge by key. A scalar replaces the inherited value. A list replaces the inherited list as a whole, so the two lists are not concatenated. -- A naming expression replaces the inherited expression. A tag specification does the same. -- `vertex_processor: null` turns off a processor inherited from `default`. A different `processor_type` replaces the whole processor declaration, parameters included. -- A different `driver_id` drops the inherited `custom_settings`. -- `false`, `0`, `[]`, and `""` are values you set on purpose. Leaving the key out is a different choice. +Use actual VideoIPath ids and factory labels in place of the synthetic values. +For a new device, omit `inventory_id`, supply the normal onboarding facts and +use a blueprint with Inventory configuration and the default `scope="all"`. + +`local` can also be a direct selector such as +`PortSelector(port_id="device1.dev.0.port-1")`; no mapping is needed then. +The selected configuration still needs a topology section (`topology: {}` is +sufficient). `scope="inventory"` skips all connection work. + +`PortSelector` takes exactly one of `port_id` or `factory_label`, with optional +`kind` to select vertices of a particular kind. A peer can use `ModuleTarget` +to restrict selection to one module. Factory labels are exact matches, not +editable display labels. Candidates in `port_mapping` are tried in order; +an ambiguous candidate fails immediately. Local factory labels can contain +`{module.position}`. Mapping keys must not duplicate `ip_vertex_mapping` keys; +either mapping's keys may be used as `local`. + +Schema-v2 inputs work at selector value, selector object, candidate list or +whole mapping positions. Overlays merge mapping keys and replace candidate +lists. Input and literal operators retain their existing atomic replacement +semantics. See [the runnable example](../../examples/07_provisioning/03_external_connections.py) +and [its input-enabled blueprint](../../examples/07_provisioning/external-connections.yml). + +| Direction | Behavior relative to the local device | +|---|---| +| `auto` (default) | Create the possible Out→In directions: one edge for a directed port pair, two when both directions exist. | +| `outgoing` | Local Out → peer In. | +| `incoming` | Peer Out → local In. | +| `bidirectional` | Require and create both directions. | + +This works with IP and directed media ports. Actual discovered vertex ids are +used, including ids without `.in`/`.out` suffixes. Invalid or ambiguous +directions fail; internal device structure remains driver-owned. + +`EdgePatch` applies the same specified fields to each requested direction: +`label`, `description`, `weight`, `capacity`, `bandwidth`, `redundancy_mode`, +`conflict_priority`, `include_formats`, `exclude_formats`, +`bandwidth_weight_factor`, `weight_per_service`, `active`, and `tags`. +Omitted fields remain unmanaged, explicit `null` is rejected, and tags support +replacement lists or `TagDelta`. Use two directed connection declarations when +opposite directions need different settings. Repeated declarations of the same +directed edge combine compatible fields and reject conflicting requirements. + +Missing edges are created; existing edges are updated only where managed values +differ. Other edges are never removed, including when `connections=[]`. +Device, vertex and edge changes share the topology transaction. Plans check +peer/edge baselines and uncommitted edit overlap before writing, and verify +written edge fields afterward. + +Missing peer devices, modules or ports remain `deferred` while other work can +finish. `fully_resolved` is false; real apply returns `partial` with +`replan_required=True`, even if no writes were needed. Create a **new plan** when +the peer appears. The engine neither provisions nor synchronizes peer devices. +A selector miss among discovered ports and API failures are errors, not deferred +discovery. A dry run reports planned/deferred work without writing. -Per-device `inventory_overrides` are applied after the variant. +## 4. Variants -The loader rejects `extends`, imports, YAML anchors, and merge keys. Each document -is complete by itself. When you need to preprocess one, build a dictionary in -Python and pass it to `Blueprint.from_dict`. +`variants=["secure", "receiver"]` selects an ordered sequence of overlays. +The default is an empty sequence, which uses `defaults` directly. An overlay +may patch Inventory, Topology, or both, and may introduce an absent section. +Unknown or repeated names fail; strings are not accepted as a variant sequence. +The library never selects a variant from source-system facts. + +Resolution order ([ADR-007](./decisions/007-typed-inputs.md)): + +1. Copy `defaults`. +2. Merge selected variants in order, with later values winning. +3. Bind typed inputs in the effective selected scope. +4. Apply the device's `inventory_overrides`. +5. Validate driver settings, processor parameters, and naming. + +Mappings merge by key; scalars and whole lists replace inherited values. Each +naming expression and tag specification replaces as a whole. References and +literal escapes are atomic during merging; their resulting objects are not +merged back into inherited objects. `vertex_processor: null` disables an +inherited processor. Changing its `processor_type` replaces the declaration +and parameters. Changing `driver_id` discards inherited `custom_settings`. +`false`, `0`, `[]`, and `""` remain explicit values. + +Every leaf retains its YAML origin (`defaults`, `variant:`, or `instance`). +A referenced value additionally records `|input:`. Source documents and +caller input objects are copied. A plan captures the selected variant order, +effective input values, processor registry, naming, and options for later apply. ## 5. Loading and validation @@ -234,8 +408,9 @@ many calls. | `Blueprint.from_yaml(text, *, source=None)` | YAML text or bytes. `source` names it in error messages. | | `Blueprint.from_dict(mapping, *, source=None)` | An already-parsed mapping. | -These check the document structure only. They do not connect to a server. -`blueprint.variant_names("inventory" | "topology")` lists the variants. +Loading checks document structure, input definitions, and references in all +variants without contacting a server. `blueprint.variant_names()` lists the +common overlay names; there is no named default variant. Files are loaded once per engine call. A plan retains the loaded configuration, so `plan.apply()` does not read the file again even if it has changed or been @@ -251,28 +426,31 @@ Timestamps stay strings. A document may be at most 1 MiB, 32 levels deep, and ```text blueprint.yml:24:9 -topology.receiver.vertex_processor.params.redundant_streams +variants.receiver.topology.vertex_processor.params.redundant_streams Expected a boolean; received a different type. ``` -`engine.validate(blueprint)`, or `blueprint.validate_full(registry)`, resolves -every variant, including variants you are not about to apply, and reports -processors that are not registered. Planning only needs the processors for the -scope you selected, so an Inventory-only run works without a topology plugin. +`engine.validate(blueprint, inputs=...)`, or +`blueprint.validate_full(registry, inputs=...)`, checks the base and every +individual variant. Supplying `variants=[...]` checks that specific ordered +combination instead. Required inputs must be provided unless they have defaults +or are replaced in that combination. Arbitrary combinations are checked at +planning time, before server access. Validation also reports unregistered +processors. An Inventory-only plan works without a Topology plugin. -`BlueprintValidationError.issues` holds `ValidationIssue` records (`path`, +`ProvisioningValidationError.issues` holds `ValidationIssue` records (`path`, `message`, `code`, `source`, `line`, `column`). Issues carry a path, not the raw value, so secrets do not leak into errors. For editor support, use the packaged JSON Schema from `published_json_schema()`, -or the file `videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json`. +or the file `videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json`. It describes the built-in processors' parameters and accepts any other `processor_type` with unconstrained `params`, so it also works with processors registered elsewhere. When you use custom processors, export `engine.json_schema()` instead. It limits `processor_type` to the engine's registered processors (a variant may still omit -it to inherit the `default` processor) and validates each one's `params`, so +it to inherit a processor from `defaults`) and validates each one's `params`, so editors flag unknown or misspelled processors and invalid parameters. `Blueprint.json_schema(registry, restrict_processor_types=True)` does the same without an engine; without the flag, only the parameter schemas are added. @@ -297,11 +475,12 @@ Available fields: | Field | Source | |---|---| -| `device.key`, `device.label`, `device.description` | `BlueprintDevice` | -| `module.position` | `BlueprintDevice.module_position` | +| `device.key`, `device.label`, `device.description` | `ProvisioningDevice` | +| `module.position` | `ProvisioningDevice.module_position` | | `endpoint.direction`, `endpoint.media`, `endpoint.index`, `endpoint.engine`, `endpoint.program`, `endpoint.leg` | The processor's `EndpointIdentity` | | `vertex.factory_label`, `vertex.id` | The discovered vertex | -| `attributes.` | Scalar leaves of `BlueprintDevice.attributes` | +| `attributes.` | Scalar leaves of `ProvisioningDevice.attributes` | +| `inputs.` | Scalar leaves of bound blueprint inputs, including nested object paths | An expression is a template string, or a tree of blocks: @@ -326,7 +505,7 @@ naming: ``` ```python -from videoipath_automation_tool.blueprints import Field, Join, NamingScheme +from videoipath_automation_tool.provisioning import Field, Join, NamingScheme scheme = NamingScheme( endpoint_label=Join( @@ -370,15 +549,14 @@ to make them unique. ## 7. Planning and applying ```python -from videoipath_automation_tool.blueprints import ApplyOptions +from videoipath_automation_tool.provisioning import ApplyOptions plan = engine.plan( device, blueprint, scope="all", # "all" | "inventory" | "topology" - inventory_variant="default", - topology_variant="receiver", - options=ApplyOptions(sync="add_only", discovery_timeout=60), + variants=["receiver"], + options=ApplyOptions(sync="add_only", topology_ready_timeout=60), ) ``` @@ -387,8 +565,10 @@ plan = engine.plan( | Field | Default | Meaning | |---|---|---| | `sync` | `"add_only"` | Topology synchronization policy (below). | -| `discovery_timeout` | `30.0` | Seconds to wait for discovery during apply. | -| `poll_interval` | `1.0` | Seconds between discovery polls. | +| `inventory_ready_timeout` | `10.0` | Seconds to wait for Inventory to report `reachable=True` before deferred topology rollout. | +| `topology_ready_timeout` | `60.0` | Independent seconds for topology addition, synchronization, and required ports/vertices. Starts after the Inventory gate. | +| `require_reachable` | `True` | Set `False` to bypass the Inventory gate explicitly, for example for mock/static devices. | +| `poll_interval` | `1.0` | Seconds between readiness polls, capped to the remaining stage budget. | | `naming_collisions` | `"reject"` | `"allow"` lets duplicate endpoint labels through. | | `write_credentials` | `False` | Write the supplied credentials to an existing record even when nothing else changes (password rotation). | @@ -399,14 +579,37 @@ before and after values (secrets redacted), where each value came from `override:`), the interface bindings, and any diagnostics. Apply then runs the phases in this order -([ADR-002](./decisions/002-plan-then-apply.md)): +([ADR-008](./decisions/008-staged-readiness.md)): 1. `inventory` creates or updates the record. A new id is stored on the result immediately. -2. `discovery` waits while the topology is not ready yet (`TopologyNotReadyError`), bounded by `discovery_timeout`. Other errors fail at once. -3. `topology_sync` adds the device to the topology, or synchronizes it. -4. `topology` edits the device and its vertices in one Inspect transaction, with conflict checking. -5. `module_tags` assigns or unassigns tags as separate actions. -6. `verification` reads the device back. `result.verification` is `confirmed` or `unconfirmed`. +2. `inventory_readiness` polls one fresh Inventory status at a time until `reachable=True`, bounded by `inventory_ready_timeout`. +3. `topology_sync` adds the device to the topology, confirms its visibility, and completes permitted synchronization. +4. `discovery` waits for the required topology data (`TopologyNotReadyError`). It shares `topology_ready_timeout` with `topology_sync`. +5. `topology` edits the device, its vertices and resolved external edges in one Inspect transaction, with conflict checking. +6. `module_tags` assigns or unassigns tags as separate actions. +7. `verification` reads the device back. `result.verification` is `confirmed` or `unconfirmed`. + +Both timeouts and `poll_interval` must be positive and finite. The Inventory wait +does not consume the topology budget. Polling starts immediately, includes read +time, and starts no further polls or rollout actions after the deadline. An +in-flight request retains its connector-level HTTP timeout; these are readiness +budgets, not a hard total duration for the whole apply. + +The Inventory gate applies to deferred rollout with an Inventory record. It uses +the created/bound record, or the physical topology target's Inventory record when +there is no source binding. Inventory-only applies, fully resolved topology, +virtual targets, and module-only edits without their own Inventory binding skip +the gate. `require_reachable=False` also reports the gate as `skipped` and performs +no status requests. Dry runs report pending readiness as `deferred` without polling. + +Unavailable status and `reachable=False` remain pending until the Inventory +deadline. Expiry raises `InventoryNotReadyError` through `ProvisioningApplyError`. +Topology expiry raises `TopologyNotReadyError`; the failed phase is +`topology_sync` or `discovery`, depending on what stalled. Messages include the +device ID, stage, timeout, and last condition. Transport, status validation, +unsupported processor configurations, and failed synchronization are errors, +not retryable readiness conditions. Completed operations and known IDs remain +on the result. A phase with nothing to do is skipped. If the whole apply would change nothing, it writes nothing. @@ -414,14 +617,22 @@ it writes nothing. ### Deferred topology work Sometimes the topology edits depend on work that has not happened yet: a record -this plan creates, a device that is not in the topology yet, or a synchronization -that is still pending. The plan marks that work `deferred`, and `plan.fully_resolved` -is `False`. During `apply()` the engine polls for discovery up to -`discovery_timeout`, then computes the exact edits from the blueprint, +this plan creates, a device that is not in the topology yet, a synchronization +that is still pending, or required local topology data that has not appeared. +The plan marks that work `deferred`, and `plan.fully_resolved` is `False`. +During `apply()` the engine first checks Inventory readiness, then uses a fresh +`topology_ready_timeout` budget to compute the exact edits from the blueprint, parameters, naming, and target captured on the plan. Those edits are recorded on the result. If that later step fails, the earlier phases stay applied, and the result says so. +An accepted topology addition is not resubmitted while waiting for visibility. +An accepted synchronization with the same pending changes is polled; newly +reported changes can trigger another permitted synchronization. A retry after +partial discovery binds the known Inventory ID and can plan against incomplete +local topology without recreating the record. Missing peer connections still +require a new plan when the peers become available. + An Inventory update that changes address, alternate addresses, credentials, generic or custom settings, or `active` is different. The plan marks the topology phases `deferred` with `fully_resolved` false, but `apply()` stops after @@ -435,14 +646,12 @@ If you want to see every write before it happens, split the scopes. Apply then build a fully resolved `scope="topology"` plan, read it, and call `plan.apply()`. -### Dry runs +### Previewing changes -`plan.apply(dry_run=True)` runs the same stale-plan, conflict, and pending-edit -checks, then returns before any write. Deferred topology stays `deferred`, -because those edits depend on writes the dry run does not perform. A -topology-affecting update also stays `deferred`, with `replan_required` set. -`status` is `planned` when writes would have run, and `no_change` otherwise. You -can dry-run a plan and apply that same plan afterwards. +Use `engine.plan(...)` and `plan.summary()` to review changes without writing. +Phases that depend on earlier writes remain `deferred`. Call `plan.apply()` +to execute the reviewed plan; it re-checks current state before the relevant +writes, as described under [Conflict checks](#conflict-checks). ### Synchronization @@ -454,7 +663,7 @@ can dry-run a plan and apply that same plan afterwards. A `syncDevices` failure is an error and is not retried. Adding a device that is not discovered yet stays `TopologyNotReadyError` and is retried until -`discovery_timeout`. +`topology_ready_timeout`. Service conflicts fail the apply. The engine does not invalidate or cancel services. A module target does not add its parent device to the topology and @@ -465,7 +674,7 @@ needs a sync, the plan reports a diagnostic. Before writing, the engine reads the Inventory record and the scoped topology again. If anything the plan relied on has changed, it raises -`BlueprintConflictError`. An Inventory update also repeats the label and address +`ProvisioningConflictError`. An Inventory update also repeats the label and address conflict check. A response with no address item list fails that check. Build a new plan. The Inspect transaction then checks its own baseline, and module tags are read again immediately before their phase. @@ -474,21 +683,27 @@ remains. Uncommitted edits on `app.inspect` that overlap the plan are rejected, so commit or discard them first. Pending edits outside the plan are left as they are. +A dry run (`plan.apply(dry_run=True)` or `engine.apply(..., dry_run=True)`) +runs the stale-plan, conflict, and pending-edit checks, then returns before writes. +Write phases report `planned`; work dependent on writes remains `deferred`. +`status` is `planned` when changes would run, or `no_change` otherwise. A dry run +does not consume the plan, so that same reviewed plan can subsequently apply. + ## 8. Results, errors, and recovery `plan.apply()` returns an `ApplyResult` when `status` is `succeeded`, -`no_change`, or `partial` with `replan_required` (`planned` on a dry run). On -failure it raises `BlueprintApplyError`. The exception's `.result` has `status` +`no_change`, `planned` (dry run), or `partial` with `replan_required`. On +failure it raises `ProvisioningApplyError`. The exception's `.result` has `status` `failed` (nothing was written), `partial` (something was written), or `unknown` (a write raised and the client cannot prove the server rejected it). The original error is `__cause__`. ```python -from videoipath_automation_tool.blueprints import BlueprintApplyError +from videoipath_automation_tool.provisioning import ProvisioningApplyError try: result = engine.apply(device, blueprint) -except BlueprintApplyError as error: +except ProvisioningApplyError as error: result = error.result if result.inventory_id: # created before the failure: bind it, never create again save_binding(device.key, result.inventory_id) @@ -498,38 +713,39 @@ except BlueprintApplyError as error: What you do next depends on how far the apply got: - An Inventory update may change the discovered topology. `status` is `partial` and `replan_required` is true. Apply did not raise. Plan again after rediscovery. -- The record was created, then discovery timed out. `status` is `partial` and `inventory_id` is set. Retry with `BlueprintDevice(..., inventory_id=result.inventory_id)`. +- The record was created, then discovery timed out. `status` is `partial` and `inventory_id` is set. Retry with `ProvisioningDevice(..., inventory_id=result.inventory_id)`. - Topology committed, then a module tag failed. `status` is `partial`. The result lists the committed edits and the tag operations that finished. Replan to finish the rest. - A write timed out. `status` is `unknown`. Read the server back before you retry. Building a new plan does that read. - Verification could not confirm the write, or a later phase failed before the topology read-back ran. The write stays applied, and `verification` is `"unconfirmed"`. The engine does not roll a partial apply back. -All errors derive from `BlueprintError`: +All errors derive from `ProvisioningError`: | Error | Raised when | |---|---| -| `BlueprintValidationError` | A document, naming expression, or parameter is invalid. `.issues` lists `ValidationIssue`s; `.codes` lists their codes. | -| `BlueprintTargetError` | A binding is missing, ambiguous, or outside the target scope. | +| `ProvisioningValidationError` | A document, naming expression, or parameter is invalid. `.issues` lists `ValidationIssue`s; `.codes` lists their codes. | +| `InventoryNotReadyError` | The Inventory readiness deadline expired before the device became reachable. | +| `ProvisioningTargetError` | A binding is missing, ambiguous, or outside the target scope. | | `ProcessorInputError` | The discovered topology is unsupported. | | `TopologyNotReadyError` | The discovered topology is still incomplete. Retried while topology work is deferred. | -| `BlueprintCapabilityError` | The engine cannot perform the requested operation. | -| `BlueprintConflictError` | The plan is stale: the server changed since planning. | -| `BlueprintApplyError` | An apply failed. `.result` is the `ApplyResult`. | +| `ProvisioningCapabilityError` | The engine cannot perform the requested operation. | +| `ProvisioningConflictError` | The plan is stale: the server changed since planning. | +| `ProvisioningApplyError` | An apply failed. `.result` is the `ApplyResult`. | ## 9. Engine configuration ```python -engine = BlueprintEngine( +engine = ProvisioningEngine( app, processors={"example-org.simple-video": SimpleVideoProcessor}, - options=ApplyOptions(sync="add_only", discovery_timeout=60), + options=ApplyOptions(sync="add_only", topology_ready_timeout=60), naming=NamingScheme(endpoint_label="{device.label}-{endpoint.direction}-{endpoint.index:02d}"), ) # equivalent: -engine = BlueprintEngine(app) +engine = ProvisioningEngine(app) engine.register_processor("example-org.simple-video", SimpleVideoProcessor) -engine.configure(options=ApplyOptions(sync="add_only", discovery_timeout=60)) +engine.configure(options=ApplyOptions(sync="add_only", topology_ready_timeout=60)) engine.configure(naming=NamingScheme(endpoint_label="{device.label}-{endpoint.direction}-{endpoint.index:02d}")) ``` @@ -542,7 +758,7 @@ naming passed to `plan()` apply to that plan only. Two engines can share one app and still keep separate registries. Sharing the app does not make concurrent writes to the same server safe. -Engine logs go to the `videoipath_automation_tool_blueprints` logger unless you +Engine logs go to the `videoipath_automation_tool_provisioning` logger unless you pass `logger=`. ## 10. Source-system adapters @@ -551,12 +767,12 @@ The library does not depend on NetBox, or on any other source system. An adapter is an ordinary function you write: ```python -from videoipath_automation_tool.blueprints import BlueprintDevice, Credentials, ModuleTarget +from videoipath_automation_tool.provisioning import ProvisioningDevice, Credentials, ModuleTarget -def to_blueprint_device(record: dict) -> BlueprintDevice: +def to_provisioning_device(record: dict) -> ProvisioningDevice: """Translate one source record (already selected and policy-checked by the caller).""" - return BlueprintDevice( + return ProvisioningDevice( key=str(record["id"]), label=record["name"], inventory_id=record.get("videoipath_inventory_id"), @@ -573,7 +789,7 @@ def to_blueprint_device(record: dict) -> BlueprintDevice: You keep the decisions around the engine: which blueprint and variant to use, which address role is allowed, where credentials come from, where returned ids -are stored, how cables are built, and when the job runs. +are stored, which concrete connections to supply, and when the job runs. ## 11. Limits and verification status @@ -583,19 +799,52 @@ are stored, how cables are built, and when the job runs. - Endpoint label uniqueness is checked per device, not across the whole system. - Vertex control configuration waits until its Inspect mapping has been verified. - Offline tests cover the feature with synthetic Inspect and Inventory data. Try the writes you care about on a test system before you use them in production. -- Also outside this feature: picking a processor from the driver automatically, pruning old configuration, rolling back a partial apply, migrating drivers, inferring cables or edges, provisioning services, and scheduling a fleet. +- Also outside this feature: picking a processor from the driver automatically, pruning old configuration, rolling back a partial apply, migrating drivers, inferring physical cabling, provisioning services, and scheduling a fleet. ## 12. Public API index -Everything below is importable from `videoipath_automation_tool.blueprints`. +Everything below is importable from `videoipath_automation_tool.provisioning`. | Area | Names | |---|---| -| Engine | `BlueprintEngine`, `BlueprintPlan`, `BlueprintApp` | -| Document | `Blueprint`, `InventorySettings`, `CatalogId`, `published_json_schema` | -| Device facts | `BlueprintDevice`, `Credentials`, `AlternativeAddress`, `DeviceTarget`, `ModuleTarget` | -| Patches | `DevicePatch`, `ModulePatch`, `VertexPatch`, `Coordinates`, `TagDelta` | +| Engine | `ProvisioningEngine`, `ProvisioningPlan`, `ProvisioningApp` | +| Document | `Blueprint`, `BlueprintConfiguration`, `InputDefinition`, `InputReference`, `LiteralValue`, `InventorySettings`, `CatalogId`, `published_json_schema` | +| Device facts | `ProvisioningDevice`, `Credentials`, `AlternativeAddress`, `DeviceTarget`, `ModuleTarget` | +| Connections | `ProvisioningConnection`, `PeerEndpoint`, `PortSelector`, `ConnectionState`, `PortBinding` | +| Patches | `DevicePatch`, `ModulePatch`, `VertexPatch`, `EdgePatch`, `Coordinates`, `TagDelta` | | Options and results | `ApplyOptions`, `ApplyResult`, `PhaseResult`, `PlannedPhase`, `PlannedOperation`, `FieldChange`, `InterfaceBinding`, `Diagnostic` | | Naming | `NamingScheme`, `DEFAULT_NAMING`, `Text`, `Field`, `Join`, `NameContext`, `NameRenderer` | | Processors | `VertexProcessor`, `ProcessorRegistry`, `ProcessingContext`, `ProcessorResult`, `VertexEdit`, `EndpointIdentity`, `SourceFacts`, `DriverContext`, `DeviceRecord`, `ModuleRecord`, `PortRecord`, `VertexRecord`, `MatroxConvertIPProcessor`, `MatroxConvertIPParams` | -| Errors | `BlueprintError`, `BlueprintValidationError`, `ValidationIssue`, `BlueprintTargetError`, `ProcessorInputError`, `TopologyNotReadyError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | +| Errors | `ProvisioningError`, `ProvisioningValidationError`, `ValidationIssue`, `ProvisioningTargetError`, `ProcessorInputError`, `InventoryNotReadyError`, `TopologyNotReadyError`, `ProvisioningCapabilityError`, `ProvisioningConflictError`, `ProvisioningApplyError` | + +## 13. Migrating from the former API + +The declarative configuration layer is now called **Provisioning**. A **Blueprint** +remains a reusable configuration template within it. Update Python imports and +call sites as follows; the former package and class names have no compatibility +aliases. + +| Former API | Current API | +|---|---| +| `videoipath_automation_tool.blueprints` (including submodules) | `videoipath_automation_tool.provisioning` | +| `BlueprintEngine` | `ProvisioningEngine` | +| `BlueprintPlan` | `ProvisioningPlan` | +| `BlueprintApp` | `ProvisioningApp` | +| `BlueprintDevice` | `ProvisioningDevice` | +| `BlueprintConnection` | `ProvisioningConnection` | +| `BlueprintError`, `BlueprintValidationError`, `BlueprintTargetError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | The same names with the `Provisioning` prefix | +| Logger `videoipath_automation_tool_blueprints` | `videoipath_automation_tool_provisioning` | + +`Blueprint`, its configuration and naming models, and the `blueprint` arguments +and `blueprint_digest` plan field keep their names. Documents use +`schema_version: 1`. The packaged schema is +`videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json`. + +### Readiness options + +`ApplyOptions.discovery_timeout` has been removed without an alias. Use +`inventory_ready_timeout=10.0` and `topology_ready_timeout=60.0` instead. Set +`require_reachable=False` explicitly for mock/static devices that do not report +reachability. The new `inventory_readiness` phase precedes `topology_sync`, and +`discovery` now follows synchronization. Consumers of phase lists should use +phase names rather than positional indexes. diff --git a/docs/examples/07_blueprints/matrox-convertip.yml b/docs/examples/07_blueprints/matrox-convertip.yml deleted file mode 100644 index b07f73e..0000000 --- a/docs/examples/07_blueprints/matrox-convertip.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Example blueprint for Matrox ConvertIP devices (synthetic tag ids). -# Validate offline: BlueprintEngine(app).validate(path) -schema_version: 1 - -inventory: - default: - driver_id: com.nevion.NMOS_multidevice-0.1.0 - generic_settings: - enable_https: false - http_auth: 0 - custom_settings: - port: 8080 - disable_rx_sdp_with_null: true - indices_in_ids: false - -topology: - default: - device: - icon_size: medium - ip_vertex_mapping: - stream-a: [P1] - stream-b: [P2] - vertex_processor: - processor_type: matrox.convertip.default - params: - redundant_streams: true - video_receiver_tags: [Format~~video-tag-a] - video_sender_tags: [Format~~video-tag-b] - audio_receiver_tags: [Format~~audio-tag-a] - audio_sender_tags: [Format~~audio-tag-b] - - receiver: - vertex_processor: - params: - mode: rx - - four-split: - vertex_processor: - params: - mode: four_split - video_receiver_tags: [Format~~video-tag-c] diff --git a/docs/examples/07_blueprints/01_onboard_with_blueprint.py b/docs/examples/07_provisioning/01_onboard_with_blueprint.py similarity index 50% rename from docs/examples/07_blueprints/01_onboard_with_blueprint.py rename to docs/examples/07_provisioning/01_onboard_with_blueprint.py index 82c657b..83e6603 100644 --- a/docs/examples/07_blueprints/01_onboard_with_blueprint.py +++ b/docs/examples/07_provisioning/01_onboard_with_blueprint.py @@ -2,11 +2,11 @@ Description ----------- -Loads ``matrox-convertip.yml``, describes one device with neutral instance facts, previews the plan, -and applies it (as a dry run by default). On the first run the Inventory record is created and the topology work is deferred -until the driver has discovered the device; later runs bind the stored Inventory id and are no-ops -when nothing changed. A partial failure still reports the created id so a retry never creates a -duplicate. +Loads ``matrox-convertip.yml``, describes one device with neutral instance facts, and previews the plan +without writing. Uncomment the apply section after reviewing the changes. On the first apply the +Inventory record is created and topology work waits until the driver has discovered the device; +later runs bind the stored Inventory id and are no-ops when nothing changed. A partial failure still +reports the created id so a retry never creates a duplicate. Prerequisites ------------- @@ -23,12 +23,11 @@ from pathlib import Path from videoipath_automation_tool import VideoIPathApp -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( ApplyOptions, - BlueprintApplyError, - BlueprintDevice, - BlueprintEngine, Credentials, + ProvisioningDevice, + ProvisioningEngine, ) SERVER_ADDRESS = "" @@ -36,17 +35,20 @@ PASSWORD = "" BLUEPRINT = Path(__file__).with_name("matrox-convertip.yml") -DRY_RUN = True # set to False to write to the server STORED_INVENTORY_ID: str | None = None # e.g. "device12" once known (persist it in your source system) def main() -> None: # --- 1. Connect and build the engine --------------------------------------- app = VideoIPathApp(server_address=SERVER_ADDRESS, username=USERNAME, password=PASSWORD, use_https=False) - engine = BlueprintEngine(app, options=ApplyOptions(sync="add_only", discovery_timeout=60)) + engine = ProvisioningEngine( + app, + options=ApplyOptions(sync="add_only", inventory_ready_timeout=10, topology_ready_timeout=60), + ) + # For mock/static devices only: set require_reachable=False to skip the Inventory gate. # --- 2. Describe the device with neutral facts ----------------------------- - device = BlueprintDevice( + device = ProvisioningDevice( key="device-a", label="device-a", description="studio-a", @@ -56,23 +58,26 @@ def main() -> None: ) # --- 3. Load the blueprint internally and preview -------------------------- - plan = engine.plan(device, BLUEPRINT, topology_variant="receiver") + plan = engine.plan(device, BLUEPRINT, variants=["receiver"]) print(plan.summary()) - if not plan.has_changes: - return # --- 4. Apply and handle partial outcomes ---------------------------------- - try: - result = plan.apply(dry_run=DRY_RUN) # a dry run checks everything but writes nothing - except BlueprintApplyError as error: - result = error.result - print(f"Apply {result.status}; known Inventory id: {result.inventory_id}") - for phase in result.phases: - print(f" {phase.name}: {phase.status} {phase.message or ''}") - raise - print(f"{result.status}: inventory={result.inventory_id} verification={result.verification}") - for binding in result.interface_bindings: - print(f" {binding.key} -> {binding.port_id}") + # Uncomment the following block to execute the reviewed plan. + # from videoipath_automation_tool.provisioning import ProvisioningApplyError + # + # if not plan.has_changes: + # return + # try: + # result = plan.apply() + # except ProvisioningApplyError as error: + # result = error.result + # print(f"Apply {result.status}; known Inventory id: {result.inventory_id}") + # for phase in result.phases: + # print(f" {phase.name}: {phase.status} {phase.message or ''}") + # raise + # print(f"{result.status}: inventory={result.inventory_id} verification={result.verification}") + # for binding in result.interface_bindings: + # print(f" {binding.key} -> {binding.port_id}") if __name__ == "__main__": diff --git a/docs/examples/07_blueprints/02_custom_processor_and_naming.py b/docs/examples/07_provisioning/02_custom_processor_and_naming.py similarity index 82% rename from docs/examples/07_blueprints/02_custom_processor_and_naming.py rename to docs/examples/07_provisioning/02_custom_processor_and_naming.py index 3ab0837..53ea61c 100644 --- a/docs/examples/07_blueprints/02_custom_processor_and_naming.py +++ b/docs/examples/07_provisioning/02_custom_processor_and_naming.py @@ -2,10 +2,11 @@ Description ----------- -Defines a tiny processor for a single-sender device, registers it on one engine, and applies a -topology-only blueprint (written as a dictionary) to an existing Inspect device. The endpoint label +Defines a tiny processor for a single-sender device, registers it on one engine, and previews a +topology-only blueprint (written as a dictionary) for an existing Inspect device. The endpoint label convention (``-EP--``) is a ``NamingScheme``, -so it can change without touching the processor. Also shows a module-scoped call. +so it can change without touching the processor. Also shows a module-scoped call. Uncomment the apply +section after reviewing the changes to write them to the server. Prerequisites ------------- @@ -23,10 +24,8 @@ from pydantic import BaseModel, ConfigDict from videoipath_automation_tool import VideoIPathApp -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( Blueprint, - BlueprintDevice, - BlueprintEngine, DeviceTarget, EndpointIdentity, Field, @@ -35,6 +34,8 @@ NamingScheme, ProcessingContext, ProcessorResult, + ProvisioningDevice, + ProvisioningEngine, VertexEdit, VertexPatch, VertexProcessor, @@ -46,7 +47,6 @@ DEVICE_ID = "device12" # existing Inspect device MODULE_ID: str | None = None # e.g. "device12.dev.1" to configure one module only -DRY_RUN = True # set to False to write to the server class SimpleParams(BaseModel): @@ -84,8 +84,8 @@ def process(self, context: ProcessingContext, params: SimpleParams) -> Processor BLUEPRINT = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "vertex_processor": { "processor_type": "example-org.simple-video", "params": {"factory_label": "Video Sender"}, @@ -97,17 +97,19 @@ def process(self, context: ProcessingContext, params: SimpleParams) -> Processor def main() -> None: app = VideoIPathApp(server_address=SERVER_ADDRESS, username=USERNAME, password=PASSWORD, use_https=False) - engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}, naming=NAMING) + engine = ProvisioningEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}, naming=NAMING) target = ModuleTarget(device_id=DEVICE_ID, module_id=MODULE_ID) if MODULE_ID else DeviceTarget(device_id=DEVICE_ID) - device = BlueprintDevice( + device = ProvisioningDevice( key="device-a", label="device-a", topology=target, module_position="1" if MODULE_ID else None ) plan = engine.plan(device, Blueprint.from_dict(BLUEPRINT), scope="topology") print(plan.summary()) # e.g. label: '' -> 'device-a-E0P1-TX20-01' - if plan.has_changes: - print(plan.apply(dry_run=DRY_RUN).status) + + # Uncomment the following block to execute the reviewed plan. + # if plan.has_changes: + # print(plan.apply().status) if __name__ == "__main__": diff --git a/docs/examples/07_provisioning/03_external_connections.py b/docs/examples/07_provisioning/03_external_connections.py new file mode 100644 index 0000000..e4c6be3 --- /dev/null +++ b/docs/examples/07_provisioning/03_external_connections.py @@ -0,0 +1,55 @@ +"""Preview connections between existing devices using a reusable port mapping. + +Connection facts and edge settings belong to the device instance. The blueprint +maps reusable local port names. Set VideoIPath credentials through the normal +VIPAT_* environment variables and replace the synthetic device ids/port labels. +Only planning runs by default; uncomment plan.apply() to execute the changes. +""" + +from __future__ import annotations + +from pathlib import Path + +from videoipath_automation_tool import VideoIPathApp +from videoipath_automation_tool.provisioning import ( + DeviceTarget, + EdgePatch, + PeerEndpoint, + PortSelector, + ProvisioningConnection, + ProvisioningDevice, + ProvisioningEngine, +) + + +def main() -> None: + engine = ProvisioningEngine(VideoIPathApp()) + device = ProvisioningDevice( + key="device-a", + label="device-a", + inventory_id="device1", + connections=[ + ProvisioningConnection( + local="uplink", + peer=PeerEndpoint( + target=DeviceTarget(device_id="device2"), + port=PortSelector(factory_label="port-1"), + ), + direction="auto", + fields=EdgePatch(weight=10, label="edge-a"), + ), + ], + ) + blueprint = Path(__file__).with_name("external-connections.yml") + plan = engine.plan(device, blueprint, scope="topology") + print(plan.summary()) + + # result = plan.apply() + # for connection in result.connections: + # print(connection.index, connection.status, connection.edge_ids, connection.reason) + # A missing peer gives status="partial", replan_required=True. Plan again + # when it is available. Existing unrelated edges are never removed. + + +if __name__ == "__main__": + main() diff --git a/docs/examples/07_provisioning/external-connections.yml b/docs/examples/07_provisioning/external-connections.yml new file mode 100644 index 0000000..028a833 --- /dev/null +++ b/docs/examples/07_provisioning/external-connections.yml @@ -0,0 +1,17 @@ +# Synthetic port labels; replace them with the factory labels reported by Inspect. +schema_version: 1 +inputs: + uplink_label: + type: string + default: port-1 +defaults: + topology: + port_mapping: + uplink: + - factory_label: {$input: uplink_label} +variants: + second-port: + topology: + port_mapping: + uplink: + - factory_label: port-2 diff --git a/docs/examples/07_provisioning/matrox-convertip.yml b/docs/examples/07_provisioning/matrox-convertip.yml new file mode 100644 index 0000000..118ac41 --- /dev/null +++ b/docs/examples/07_provisioning/matrox-convertip.yml @@ -0,0 +1,51 @@ +# Example blueprint for Matrox ConvertIP devices (synthetic tag ids). +# Validate offline: ProvisioningEngine(app).validate(path) +schema_version: 1 +inputs: + api_port: + type: integer + default: 8080 +defaults: + inventory: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + generic_settings: + enable_https: false + http_auth: 0 + custom_settings: + port: + $input: api_port + disable_rx_sdp_with_null: true + indices_in_ids: false + topology: + device: + icon_size: medium + ip_vertex_mapping: + stream-a: + - P1 + stream-b: + - P2 + vertex_processor: + processor_type: matrox.convertip.default + params: + redundant_streams: true + video_receiver_tags: + - Format~~video-tag-a + video_sender_tags: + - Format~~video-tag-b + audio_receiver_tags: + - Format~~audio-tag-a + audio_sender_tags: + - Format~~audio-tag-b +variants: + receiver: + topology: + vertex_processor: + params: + mode: rx + four-split: + topology: + vertex_processor: + params: + mode: four_split + video_receiver_tags: + - Format~~video-tag-c diff --git a/docs/examples/README.md b/docs/examples/README.md index d2cc12c..7d9b2f0 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -95,13 +95,16 @@ Each scenario is implemented twice — once with the modern **Inspect** app, onc - [03_bulk_retag_and_relabel.py](06_workflows/03_bulk_retag_and_relabel.py) — apply a naming/tagging policy across a fleet with a dry-run report and a single batched commit. -### 07 — Blueprints +### 07 — Provisioning -See the [Blueprint guide](../getting-started-guide/05_Blueprints.md). +See the [Provisioning guide](../getting-started-guide/05_Provisioning.md) for +declarative device configuration through the SDK. -- [matrox-convertip.yml](07_blueprints/matrox-convertip.yml) — example blueprint with Inventory +- [matrox-convertip.yml](07_provisioning/matrox-convertip.yml) — example blueprint with Inventory settings, interface mapping, the Matrox processor, and `receiver` / `four-split` variants. -- [01_onboard_with_blueprint.py](07_blueprints/01_onboard_with_blueprint.py) — preview and apply a +- [01_onboard_with_blueprint.py](07_provisioning/01_onboard_with_blueprint.py) — preview and apply a blueprint (creation with deferred discovery, idempotent re-runs, partial-failure handling). -- [02_custom_processor_and_naming.py](07_blueprints/02_custom_processor_and_naming.py) — register a +- [02_custom_processor_and_naming.py](07_provisioning/02_custom_processor_and_naming.py) — register a custom vertex processor and a custom endpoint naming convention; device or module scope. + +- [03_external_connections.py](07_provisioning/03_external_connections.py) — preview external edges supplied on a device, using [external-connections.yml](07_provisioning/external-connections.yml) for generic port mapping. diff --git a/docs/getting-started-guide/05_Blueprints.md b/docs/getting-started-guide/05_Blueprints.md deleted file mode 100644 index 53aa84a..0000000 --- a/docs/getting-started-guide/05_Blueprints.md +++ /dev/null @@ -1,195 +0,0 @@ -# 05 — Blueprint-driven Configuration - -A **blueprint** is a YAML file that describes how one *kind* of device should be -configured: its Inventory driver settings, how it appears in Inspect, and which of -its discovered vertices become named, tagged endpoints. You write it once and -apply it to every real device of that kind. - -This page covers the essentials. The complete reference (every field, option, -error, and the API index) is in -[docs/architecture/blueprints](../architecture/blueprints/README.md). - -## 1. Key concepts - -| Concept | What it is | -|---|---| -| `Blueprint` | The YAML document: an `inventory` and/or `topology` section, each with a `default` entry and optional named **variants**. | -| `BlueprintDevice` | The facts of one real device that only your source system knows: id, label, addresses, credentials. | -| `BlueprintEngine` | Combines a blueprint with a device. `plan()` previews the changes, `apply()` writes them through `app.inventory` and `app.inspect`. | -| Vertex processor | Plugin code that recognizes a driver's discovered vertices and proposes endpoint settings. A Matrox ConvertIP processor is built in. | -| Naming | Rules that build labels from device facts, so re-applying never stacks suffixes. | - -Two rules shape everything else: - -- **Plan, then apply.** Planning only reads. The plan shows every change with - before and after values, so you can review it before anything is written. -- **Only what you set is managed.** Fields the blueprint leaves out stay as they - are on the server. Removing a field from the blueprint does not reset it. - -## 2. Quick start - -```python -from videoipath_automation_tool import VideoIPathApp -from videoipath_automation_tool.blueprints import BlueprintDevice, BlueprintEngine - -app = VideoIPathApp() -engine = BlueprintEngine(app) # the app has no blueprint attribute; create the engine yourself - -device = BlueprintDevice(key="device-a", label="device-a", management_address="192.0.2.10") - -plan = engine.plan(device, "blueprints/matrox-convertip.yml") -print(plan.summary()) # review the changes - -result = plan.apply() -print(result.status, result.inventory_id) # persist inventory_id in your source system -``` - -`engine.apply(device, blueprint)` plans and applies in one call. Pass -`dry_run=True` to `apply()` to run every check without writing. The blueprint -argument is a file path or a `Blueprint` object; PyYAML ships with the package. - -On later runs, pass the stored id as `BlueprintDevice(..., inventory_id=...)` so the -engine updates that record instead of creating a new one. When nothing changed, -the apply writes nothing. - -## 3. Writing a blueprint - -```yaml -schema_version: 1 - -inventory: - default: - driver_id: com.nevion.NMOS_multidevice-0.1.0 - generic_settings: {enable_https: false, http_auth: 0} - custom_settings: {port: 8080} - -topology: - default: - device: {icon_size: medium} - vertex_processor: - processor_type: matrox.convertip.default - params: - redundant_streams: true - video_sender_tags: [Format~~video-tag-b] - receiver: - vertex_processor: - params: {mode: rx} -``` - -- **`inventory`** sets the driver and its settings. `custom_settings` are checked - against the driver schema, so a typo or wrong type fails the plan before any write. -- **`topology`** sets device appearance, the vertex processor and its `params`, - and optional explicit vertex overrides. -- **Variants** such as `receiver` are patches on top of `default`. Choose them per - call with `inventory_variant=` and `topology_variant=`. -- **Tags** are exact catalog ids. A list replaces the local tags; `{add: [...], remove: [...]}` - changes only the listed ones. - -Check every variant offline with `engine.validate("blueprints/matrox-convertip.yml")`. -Errors point at the file, line, and field. For editor completion, use the -packaged JSON Schema (`published_json_schema()`), or `engine.json_schema()` when -you register custom processors (it only accepts registered `processor_type` values). - -Details: [document reference](../architecture/blueprints/reference.md#3-blueprint-document), -[variants](../architecture/blueprints/reference.md#4-variants), -[loading and validation](../architecture/blueprints/reference.md#5-loading-and-validation). - -## 4. Describing a device - -```python -from videoipath_automation_tool.blueprints import BlueprintDevice, Credentials, ModuleTarget - -device = BlueprintDevice( - key="source-id-42", # your id; never written to VideoIPath - label="device-a", - inventory_id="device12", # omit on first run to create the record - management_address="192.0.2.10", - credentials=Credentials(username="test-user", password="test-password"), - attributes={"site": "site-a"}, # extra facts for naming -) -``` - -Credentials stay out of YAML and are redacted in plans, results, and errors. To -configure one module of a chassis, pass `topology=ModuleTarget(device_id, module_id)` -with the exact Inspect ids. - -Details: [all fields and supported arrangements](../architecture/blueprints/reference.md#2-describing-a-device-blueprintdevice). - -## 5. Naming - -By default, Inventory and device labels are the device `label`, and endpoint -labels look like `device-a-TX-video-01` (or `device-a-M1-RX-audio-01` with a -module position). Override any entry in the blueprint or per call: - -```yaml -topology: - default: - naming: - endpoint_label: "{device.label}-{endpoint.direction}-{endpoint.index:02d}" -``` - -Endpoint labels must be unique on the device; a collision fails the plan. - -Details: [fields, `Text` / `Field` / `Join` blocks, layering](../architecture/blueprints/reference.md#6-naming). - -## 6. What apply does - -Apply runs these phases in order, skipping any with nothing to do: - -1. **inventory** — create or update the record. -2. **discovery** — wait for the driver to discover the device (`discovery_timeout`). -3. **topology_sync** — add the device to the topology or synchronize it. -4. **topology** — commit device and vertex edits in one Inspect transaction. -5. **module_tags** — assign or remove module tags. -6. **verification** — read back what was written. - -When the plan creates the Inventory record, the topology edits cannot be known -yet. The plan marks them `deferred` and computes them during apply, after -discovery. - -When an update changes the address, alternate addresses, credentials, generic or -custom settings, or whether the device is active, apply stops after Inventory. -The result is `partial` with `replan_required`. Plan again after the driver has -rediscovered the device. - -If the server changed since planning, apply raises -`BlueprintConflictError`; build a new plan. Behavior is tuned with -`ApplyOptions` (`sync`, `discovery_timeout`, ...). - -Details: [planning and applying](../architecture/blueprints/reference.md#7-planning-and-applying). - -## 7. Handling failures - -Apply never rolls back. If a later phase fails, the earlier ones stay applied and -`BlueprintApplyError.result` tells you how far it got: - -```python -from videoipath_automation_tool.blueprints import BlueprintApplyError - -try: - result = engine.apply(device, "blueprints/matrox-convertip.yml") -except BlueprintApplyError as error: - if error.result.inventory_id: # record was created: store it so a retry updates, not duplicates - save_binding(device.key, error.result.inventory_id) - raise -``` - -Details: [results, errors, and recovery](../architecture/blueprints/reference.md#8-results-errors-and-recovery). - -## 8. Custom processors - -When no built-in processor fits a driver, write one: a `VertexProcessor` subclass -with a Pydantic `params_model` that reads the discovered vertices and returns -proposed edits. Register it on the engine and reference its id in YAML: - -```python -engine.register_processor("example-org.simple-video", SimpleVideoProcessor) -``` - -Details: [processor contract and the Matrox built-in](../architecture/blueprints/processors.md). - -## Next steps - -- Runnable scripts: [docs/examples/07_blueprints](../examples/07_blueprints/) -- Full reference: [reference.md](../architecture/blueprints/reference.md) -- Design and decisions: [concepts.md](../architecture/blueprints/concepts.md) diff --git a/docs/getting-started-guide/05_Provisioning.md b/docs/getting-started-guide/05_Provisioning.md new file mode 100644 index 0000000..6fafc06 --- /dev/null +++ b/docs/getting-started-guide/05_Provisioning.md @@ -0,0 +1,239 @@ +# 05 — Provisioning + +**Provisioning** is a declarative configuration layer on top of the VideoIPath +SDK. It automates device configuration in an Infrastructure as Code style: +combine reusable templates with device facts, review a plan, then apply it +through the existing Inventory and Inspect apps. + +A **blueprint** is a YAML file that describes how one *kind* of device should be +configured: its Inventory driver settings, how it appears in Inspect, and which of +its discovered vertices become named, tagged endpoints. You write it once and +apply it to every real device of that kind. Blueprints are the templates used +within provisioning. + +This page covers the essentials. The complete reference (every field, option, +error, and the API index) is in +[docs/architecture/provisioning](../architecture/provisioning/README.md). +For existing Python integrations, see the +[API migration notes](../architecture/provisioning/reference.md#13-migrating-from-the-former-api). + +## 1. Key concepts + +| Concept | What it is | +|---|---| +| `Blueprint` | The v2 YAML document: typed `inputs`, Inventory/Topology `defaults`, and optional ordered variant overlays. | +| `ProvisioningDevice` | The facts of one real device that only your source system knows: id, label, addresses, credentials. | +| `ProvisioningEngine` | Combines a blueprint with a device. `plan()` previews the changes, `apply()` writes them through `app.inventory` and `app.inspect`. | +| Vertex processor | Plugin code that recognizes a driver's discovered vertices and proposes endpoint settings. A Matrox ConvertIP processor is built in. | +| Naming | Rules that build labels from device facts, so re-applying never stacks suffixes. | + +Two rules shape everything else: + +- **Plan, then apply.** Planning only reads. The plan shows every change with + before and after values, so you can review it before anything is written. +- **Only what you set is managed.** Fields the blueprint leaves out stay as they + are on the server. Removing a field from the blueprint does not reset it. + +## 2. Quick start + +```python +from videoipath_automation_tool import VideoIPathApp +from videoipath_automation_tool.provisioning import ProvisioningDevice, ProvisioningEngine + +app = VideoIPathApp() +engine = ProvisioningEngine(app) # the app has no provisioning attribute; create the engine yourself + +device = ProvisioningDevice(key="device-a", label="device-a", management_address="192.0.2.10") + +plan = engine.plan(device, "blueprints/matrox-convertip.yml") +print(plan.summary()) # review the changes + +result = plan.apply() +print(result.status, result.inventory_id) # persist inventory_id in your source system +``` + +`engine.apply(device, blueprint)` plans and applies in one call. For a read-only +preview, use `engine.plan(...)` and `plan.summary()`. Call `plan.apply()` when +you want to execute the reviewed changes. The blueprint +argument is a file path or a `Blueprint` object; PyYAML ships with the package. + +On later runs, pass the stored id as `ProvisioningDevice(..., inventory_id=...)` so the +engine updates that record instead of creating a new one. When nothing changed, +the apply writes nothing. + +### External edges + +Supply concrete connections through `ProvisioningDevice.connections`. Each +`ProvisioningConnection` contains a local port (a mapping key or `PortSelector`), +a `PeerEndpoint` identifying the other device/module and port, and optional +`EdgePatch` settings. The blueprint's `defaults.topology.port_mapping` maps +reusable local names to discovered ports. `plan(...).apply()` creates or updates +the requested edges along with device and vertex settings. + +`direction="auto"` derives one or both directions from the actual port vertices. +Existing unrelated edges remain unchanged. Missing peers are reported per +connection as `deferred`; the result is `partial` with `replan_required=True`. +Build a new plan after the peer becomes available. + +See the [complete example](../examples/07_provisioning/03_external_connections.py) +and [connection reference](../architecture/provisioning/reference.md#external-connections). + +## 3. Writing a blueprint + +```yaml +schema_version: 1 +defaults: + inventory: + driver_id: com.nevion.NMOS_multidevice-0.1.0 + generic_settings: + enable_https: false + http_auth: 0 + custom_settings: + port: 8080 + topology: + device: + icon_size: medium + vertex_processor: + processor_type: matrox.convertip.default + params: + redundant_streams: true + video_sender_tags: + - Format~~video-tag-b +variants: + receiver: + topology: + vertex_processor: + params: + mode: rx +``` + +- **`inventory`** sets the driver and its settings. `custom_settings` are checked + against the driver schema, so a typo or wrong type fails the plan before any write. +- **`topology`** sets device appearance, the vertex processor and its `params`, + and optional explicit vertex overrides. +- **Variants** such as `receiver` patch `defaults`. Choose an ordered sequence per + call with `variants=["secure", "receiver"]`; later values win. +- **Tags** are exact catalog ids. A list replaces the local tags; `{add: [...], remove: [...]}` + changes only the listed ones. + +- **Inputs** declare strict types and optional defaults. Use `port: {$input: api_port}` + and pass `inputs={"api_port": 8080}` to the engine. Naming can use scalar input + leaves such as `{inputs.location.code}`. Values can come from any caller or data source. + +Check the base and every individual variant offline with `engine.validate("blueprints/matrox-convertip.yml")`. +Use `engine.validate(path, inputs=..., variants=[...])` to check one combined +selection. Errors point at the file, line, and field. For editor completion, use the +packaged JSON Schema (`published_json_schema()`), or `engine.json_schema()` when +you register custom processors (it only accepts registered `processor_type` values). + +Details: [document reference](../architecture/provisioning/reference.md#3-blueprint-document), +[variants](../architecture/provisioning/reference.md#4-variants), +[loading and validation](../architecture/provisioning/reference.md#5-loading-and-validation). + +## 4. Describing a device + +```python +from videoipath_automation_tool.provisioning import ProvisioningDevice, Credentials, ModuleTarget + +device = ProvisioningDevice( + key="source-id-42", # your id; never written to VideoIPath + label="device-a", + inventory_id="device12", # omit on first run to create the record + management_address="192.0.2.10", + credentials=Credentials(username="test-user", password="test-password"), + attributes={"site": "site-a"}, # extra facts for naming +) +``` + +Credentials stay out of YAML and are redacted in plans, results, and errors. To +configure one module of a chassis, pass `topology=ModuleTarget(device_id, module_id)` +with the exact Inspect ids. + +Details: [all fields and supported arrangements](../architecture/provisioning/reference.md#2-describing-a-device-provisioningdevice). + +## 5. Naming + +By default, Inventory and device labels are the device `label`, and endpoint +labels look like `device-a-TX-video-01` (or `device-a-M1-RX-audio-01` with a +module position). Override any entry in the blueprint or per call: + +```yaml +defaults: + topology: + naming: + endpoint_label: "{device.label}-{endpoint.direction}-{endpoint.index:02d}" +``` + +Endpoint labels must be unique on the device; a collision fails the plan. + +Details: [fields, `Text` / `Field` / `Join` blocks, layering](../architecture/provisioning/reference.md#6-naming). + +## 6. What apply does + +Apply runs these phases in order, skipping any with nothing to do: + +1. **inventory** — create or update the record. +2. **inventory_readiness** — wait up to `inventory_ready_timeout` (10 seconds) for `reachable=True`. +3. **topology_sync** — add the device to the topology, confirm its visibility, and synchronize it. +4. **discovery** — wait for the required ports and vertices. This shares a fresh `topology_ready_timeout` (60 seconds) with topology synchronization. +5. **topology** — commit device, vertex and resolved external edge edits in one Inspect transaction. +6. **module_tags** — assign or remove module tags. +7. **verification** — read back what was written. + +When the plan creates the Inventory record, the topology edits cannot be known +yet. The plan marks them `deferred` and computes them during apply, after +discovery. + +Inventory and topology have independent budgets; each polls every `poll_interval` +(1 second by default). For mock/static devices, use +`ApplyOptions(require_reachable=False)` to bypass the Inventory gate. Inventory-only +applies and already resolved topology do not wait for reachability. Dry runs never +poll. A timeout identifies the stalled phase and retains the created Inventory ID +so a later plan can continue from that record. + +When an update changes the address, alternate addresses, credentials, generic or +custom settings, or whether the device is active, apply stops after Inventory. +The result is `partial` with `replan_required`. Plan again after the driver has +rediscovered the device. + +If the server changed since planning, apply raises +`ProvisioningConflictError`; build a new plan. Behavior is tuned with +`ApplyOptions` (`sync`, `inventory_ready_timeout`, `topology_ready_timeout`, ...). + +Details: [planning and applying](../architecture/provisioning/reference.md#7-planning-and-applying). + +## 7. Handling failures + +Apply never rolls back. If a later phase fails, the earlier ones stay applied and +`ProvisioningApplyError.result` tells you how far it got: + +```python +from videoipath_automation_tool.provisioning import ProvisioningApplyError + +try: + result = engine.apply(device, "blueprints/matrox-convertip.yml") +except ProvisioningApplyError as error: + if error.result.inventory_id: # record was created: store it so a retry updates, not duplicates + save_binding(device.key, error.result.inventory_id) + raise +``` + +Details: [results, errors, and recovery](../architecture/provisioning/reference.md#8-results-errors-and-recovery). + +## 8. Custom processors + +When no built-in processor fits a driver, write one: a `VertexProcessor` subclass +with a Pydantic `params_model` that reads the discovered vertices and returns +proposed edits. Register it on the engine and reference its id in YAML: + +```python +engine.register_processor("example-org.simple-video", SimpleVideoProcessor) +``` + +Details: [processor contract and the Matrox built-in](../architecture/provisioning/processors.md). + +## Next steps + +- Runnable scripts: [docs/examples/07_provisioning](../examples/07_provisioning/) +- Full reference: [reference.md](../architecture/provisioning/reference.md) +- Design and decisions: [concepts.md](../architecture/provisioning/concepts.md) diff --git a/docs/getting-started-guide/README.md b/docs/getting-started-guide/README.md index 34c7d03..0919d68 100644 --- a/docs/getting-started-guide/README.md +++ b/docs/getting-started-guide/README.md @@ -25,7 +25,7 @@ onboarding is unchanged and remains a prerequisite for both. - A. [Topology App](03_A_Topology.md) (classic / legacy) - B. [Inspect App](03_B_Inspect.md) (recommended on 2025.4+) 4. [Configuring Multicast Pools](04_Multicast_Pools.md) -5. [Blueprint-driven Configuration](05_Blueprints.md) — reusable YAML blueprints, planning, and custom vertex processors +5. [Provisioning](05_Provisioning.md) — declarative device configuration using YAML blueprints, planning, and custom vertex processors For runnable, task-oriented scripts covering realistic automation scenarios (including paired Topology/Inspect examples), see the diff --git a/src/videoipath_automation_tool/apps/inspect/api/inspect_api.py b/src/videoipath_automation_tool/apps/inspect/api/inspect_api.py index 0939749..22b9ee1 100644 --- a/src/videoipath_automation_tool/apps/inspect/api/inspect_api.py +++ b/src/videoipath_automation_tool/apps/inspect/api/inspect_api.py @@ -10,6 +10,7 @@ import logging from typing import Any +from videoipath_automation_tool.apps.inspect.errors import InspectEntityNotFoundError from videoipath_automation_tool.apps.inspect.model.actions import ( InspectApiAddDevicesItem, InspectApiAddDevicesRequest, @@ -152,6 +153,8 @@ def get_ngraph_factory_labels(self, *element_ids: str) -> dict[str, str]: def lookup_inspect_device(self, device_id: str) -> InspectApiLookupInspectDeviceResponse: request = InspectApiLookupInspectDeviceRequest(data=device_id) response = self.vip_connector.rest.post("/rest/v2/actions/status/collector/lookupInspectDevice", request) + if response.data is None: + raise InspectEntityNotFoundError(device_id, "device") return InspectApiLookupInspectDeviceResponse.model_validate(_post_envelope(response)) def lookup_vertices(self, vertex_ids: list[str]) -> InspectApiLookupVerticesResponse: diff --git a/src/videoipath_automation_tool/apps/inventory/errors.py b/src/videoipath_automation_tool/apps/inventory/errors.py index 68bfbd6..5a0def7 100644 --- a/src/videoipath_automation_tool/apps/inventory/errors.py +++ b/src/videoipath_automation_tool/apps/inventory/errors.py @@ -7,6 +7,10 @@ InventoryWriteOperation = Literal["add", "update"] +class InventoryStatusUnavailableError(ValueError): + """The device has no Inventory status yet; a later single-attempt read may succeed.""" + + class InventoryWriteNotAppliedError(ValueError): """An Inventory write was rejected by the server or not attempted, so it did not change the record. @@ -19,4 +23,4 @@ def __init__(self, message: str, *, operation: InventoryWriteOperation) -> None: super().__init__(message) -__all__ = ["InventoryWriteNotAppliedError", "InventoryWriteOperation"] +__all__ = ["InventoryStatusUnavailableError", "InventoryWriteNotAppliedError", "InventoryWriteOperation"] diff --git a/src/videoipath_automation_tool/apps/inventory/inventory_api.py b/src/videoipath_automation_tool/apps/inventory/inventory_api.py index f64c722..5604ba9 100644 --- a/src/videoipath_automation_tool/apps/inventory/inventory_api.py +++ b/src/videoipath_automation_tool/apps/inventory/inventory_api.py @@ -8,7 +8,10 @@ from pydantic import IPvAnyAddress from typing_extensions import deprecated -from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError +from videoipath_automation_tool.apps.inventory.errors import ( + InventoryStatusUnavailableError, + InventoryWriteNotAppliedError, +) from videoipath_automation_tool.apps.inventory.inventory_utils import ( construct_driver_id_from_info, extract_driver_info_from_id, @@ -339,7 +342,7 @@ def _fetch_device_status(self, device_id: str) -> DeviceStatus: response = self.vip_connector.rest.get(f"/rest/v2/data/status/devman/devices/* where id='{device_id}' /**") if response.data and response.data["status"]["devman"]["devices"]["_items"]: return DeviceStatus(**response.data["status"]["devman"]["devices"]["_items"][0]) - raise ValueError(f"Device with id '{device_id}' not found.") + raise InventoryStatusUnavailableError(f"Status for device '{device_id}' is not available yet.") def _fetch_device_config_by_uuid(self, uuid: str) -> InventoryDevice: """Method to receive a device configuration by uuid in Meta field from VideoIPath-Inventory diff --git a/src/videoipath_automation_tool/blueprints/resolution.py b/src/videoipath_automation_tool/blueprints/resolution.py deleted file mode 100644 index 8c9c0db..0000000 --- a/src/videoipath_automation_tool/blueprints/resolution.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Variant resolution, driver/parameter validation, and JSON Schema export. - -Resolution order per section: the ``default`` entry, then the selected named entry (a patch), then -per-instance Inventory overrides. Merge rules are small and fixed: - -- mappings merge recursively by key; scalars replace; lists replace entirely; ``{}`` adds nothing; -- naming expressions and tag specifications are replaced as a whole; -- ``vertex_processor: null`` disables an inherited processor; a different ``processor_type`` - replaces the whole processor specification (parameters are not inherited); -- a different ``driver_id`` replaces the inherited ``custom_settings``. - -The source document is never mutated. Every merged leaf records its provenance (``default``, -``variant:``, ``instance``) so plans can explain where a managed value came from. -""" - -from __future__ import annotations - -import copy -import hashlib -import importlib.resources -import json -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, ValidationError - -from videoipath_automation_tool.apps.inventory.model.drivers import ( - DRIVER_ID_TO_CUSTOM_SETTINGS, - SELECTED_SCHEMA_VERSION, -) -from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue -from videoipath_automation_tool.blueprints.models import ( - Blueprint, - InventoryConfig, - InventorySettings, - Scope, - TopologyConfig, - issues_from_pydantic, -) -from videoipath_automation_tool.blueprints.naming import NamingScheme -from videoipath_automation_tool.blueprints.processors import ProcessorRegistry, VertexProcessor - -Provenance = dict[str, str] - - -class ResolvedInventory(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - variant: str - driver_id: str - schema_version: str = SELECTED_SCHEMA_VERSION - config: InventoryConfig - custom_settings: dict[str, Any] - """Explicitly configured custom settings (Python field names), validated against the driver model.""" - provenance: Provenance - - -class ResolvedTopology(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - variant: str - config: TopologyConfig - processor_id: str | None = None - processor_cls: type[VertexProcessor] | None = None # type: ignore[type-arg] - params: BaseModel | None = None - provenance: Provenance - - -class ResolvedBlueprint(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - inventory: ResolvedInventory | None = None - topology: ResolvedTopology | None = None - naming: NamingScheme - skipped: dict[str, str] - digest: str - - -def resolve_blueprint( - blueprint: Blueprint, - *, - scope: Scope, - inventory_variant: str, - topology_variant: str, - registry: ProcessorRegistry, - overrides: InventorySettings | None = None, -) -> ResolvedBlueprint: - """Resolve the selected sections and variants; raises :class:`BlueprintValidationError`.""" - if scope not in ("all", "inventory", "topology"): - raise BlueprintValidationError(f"Invalid scope {scope!r}; expected 'all', 'inventory', or 'topology'.") - selected = { - "inventory": _select(blueprint, "inventory", inventory_variant, scope), - "topology": _select(blueprint, "topology", topology_variant, scope), - } - skipped = {section: reason for section, reason in selected.items() if reason is not None} - - inventory = None - if "inventory" not in skipped: - inventory = resolve_inventory(blueprint, inventory_variant, overrides=overrides) - topology = None - if "topology" not in skipped: - topology = resolve_topology(blueprint, topology_variant, registry=registry, require_processor=True) - - naming = NamingScheme.layered( - inventory.config.naming.to_scheme() if inventory and inventory.config.naming else None, - topology.config.naming.to_scheme() if topology and topology.config.naming else None, - ) - return ResolvedBlueprint( - inventory=inventory, topology=topology, naming=naming, skipped=skipped, digest=blueprint_digest(blueprint) - ) - - -def resolve_inventory( - blueprint: Blueprint, variant: str, *, overrides: InventorySettings | None = None -) -> ResolvedInventory: - raw = _raw_entries(blueprint, "inventory") - base = copy.deepcopy(raw["default"]) - provenance = _leaf_provenance(base, "default") - if variant != "default": - patch = raw[variant] - if "driver_id" in patch and patch["driver_id"] != base.get("driver_id"): - base.pop("custom_settings", None) - provenance = {path: src for path, src in provenance.items() if not path.startswith("custom_settings")} - base = _merge(base, patch, (), provenance, f"variant:{variant}") - if overrides is not None: - base = _merge(base, overrides.model_dump(exclude_unset=True), (), provenance, "instance") - - entry_path = ("inventory", variant) - config = _validate_entry(InventoryConfig, base, blueprint, entry_path, provenance) - if config.driver_id is None: - raise BlueprintValidationError( - _issue(blueprint, entry_path + ("driver_id",), "Field required (inventory needs a driver_id).", "missing") - ) - model = DRIVER_ID_TO_CUSTOM_SETTINGS.get(config.driver_id) - if model is None: - raise BlueprintValidationError( - _issue( - blueprint, - _source_path(entry_path, ("driver_id",), provenance, "driver_id"), - f"Unknown driver '{config.driver_id}' for the selected driver schema {SELECTED_SCHEMA_VERSION}.", - "driver.unknown", - ) - ) - custom = dict(config.custom_settings or {}) - return ResolvedInventory( - variant=variant, - driver_id=config.driver_id, - config=config, - custom_settings=_validate_custom_settings(model, custom, blueprint, entry_path, provenance, config.driver_id), - provenance=provenance, - ) - - -def resolve_topology( - blueprint: Blueprint, - variant: str, - *, - registry: ProcessorRegistry | None, - require_processor: bool, -) -> ResolvedTopology: - raw = _raw_entries(blueprint, "topology") - base = copy.deepcopy(raw["default"]) - provenance = _leaf_provenance(base, "default") - if variant != "default": - patch = copy.deepcopy(raw[variant]) - source = f"variant:{variant}" - if "vertex_processor" in patch: - base["vertex_processor"] = _merge_processor( - base.get("vertex_processor"), patch.pop("vertex_processor"), provenance, source - ) - base = _merge(base, patch, (), provenance, source) - - entry_path = ("topology", variant) - config = _validate_entry(TopologyConfig, base, blueprint, entry_path, provenance) - spec = config.vertex_processor - if spec is None: - return ResolvedTopology(variant=variant, config=config, provenance=provenance) - if spec.processor_type is None: - raise BlueprintValidationError( - _issue(blueprint, entry_path + ("vertex_processor", "processor_type"), "Field required.", "missing") - ) - if registry is None or (spec.processor_type not in registry and not require_processor): - return ResolvedTopology(variant=variant, config=config, processor_id=spec.processor_type, provenance=provenance) - - type_path = _source_path( - entry_path, ("vertex_processor", "processor_type"), provenance, "vertex_processor.processor_type" - ) - try: - processor_cls = registry.get(spec.processor_type) - except BlueprintValidationError as exc: - raise BlueprintValidationError( - _issue(blueprint, type_path, exc.issues[0].message, "processor.unknown") - ) from None - try: - params = registry.validate_params(spec.processor_type, spec.params) - except BlueprintValidationError as exc: - raise BlueprintValidationError( - [ - _relocate_relative( - blueprint, - entry_path, - issue.model_copy(update={"path": f"vertex_processor.params.{issue.path}".rstrip(".")}), - provenance, - ) - for issue in exc.issues - ] - ) from None - return ResolvedTopology( - variant=variant, - config=config, - processor_id=spec.processor_type, - processor_cls=processor_cls, - params=params, - provenance=provenance, - ) - - -def validate_all_variants(blueprint: Blueprint, registry: ProcessorRegistry | None) -> None: - """Resolve every variant of every section; collect and raise all issues.""" - issues: list[ValidationIssue] = [] - for variant in blueprint.variant_names("inventory"): - try: - resolve_inventory(blueprint, variant) - except BlueprintValidationError as exc: - issues.extend(exc.issues) - for variant in blueprint.variant_names("topology"): - try: - resolved = resolve_topology(blueprint, variant, registry=registry, require_processor=False) - except BlueprintValidationError as exc: - issues.extend(exc.issues) - continue - if registry is not None and resolved.processor_id is not None and resolved.processor_cls is None: - issues.append( - _issue( - blueprint, - ("topology", variant, "vertex_processor", "processor_type"), - f"Processor '{resolved.processor_id}' is not registered; its parameters were not validated.", - "processor.unavailable", - ) - ) - if issues: - raise BlueprintValidationError(issues) - - -def blueprint_digest(blueprint: Blueprint) -> str: - """Stable digest of the normalized document.""" - document = blueprint.model_dump(mode="json", by_alias=True, exclude_unset=True) - return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - - -def blueprint_json_schema( - registry: ProcessorRegistry | None = None, *, restrict_processor_types: bool = False -) -> dict[str, Any]: - """JSON Schema (draft 2020-12) generated from the runtime models. With ``registry``, the - ``params`` of each registered ``processor_type`` are described through ``if``/``then`` branches. - - ``restrict_processor_types`` (requires ``registry``) limits ``processor_type`` to the registered - ids, so editors flag unknown or misspelled processors. A variant may still omit it to inherit - the ``default`` processor. Without it, unknown ids are accepted and their ``params`` are - unconstrained (the packaged schema stays usable with processors registered elsewhere). - """ - if restrict_processor_types and registry is None: - raise ValueError("restrict_processor_types requires a registry.") - schema = Blueprint.model_json_schema(by_alias=True) - schema = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", - **schema, - "title": "VideoIPath Automation Tool blueprint (schema_version 1)", - } - if registry is None: - return schema - defs = schema.setdefault("$defs", {}) - branches: list[dict[str, Any]] = [] - for processor_id in registry.ids(): - params_schema = registry.get(processor_id).params_model.model_json_schema( - by_alias=True, ref_template="#/$defs/{model}" - ) - defs.update(params_schema.pop("$defs", {})) - branches.append( - { - "if": {"properties": {"processor_type": {"const": processor_id}}, "required": ["processor_type"]}, - "then": {"properties": {"params": params_schema}}, - } - ) - spec = defs["VertexProcessorSpec"] - if branches: - spec["allOf"] = branches - if restrict_processor_types: - processor_type = spec["properties"]["processor_type"] - processor_type.pop("anyOf", None) - processor_type.update({"type": "string", "enum": registry.ids()}) - return schema - - -def published_json_schema() -> dict[str, Any]: - """The packaged ``schemas/blueprint-v1.schema.json`` (document plus built-in processor parameters).""" - resource = importlib.resources.files("videoipath_automation_tool.blueprints").joinpath( - "schemas/blueprint-v1.schema.json" - ) - return json.loads(resource.read_text(encoding="utf-8")) - - -# --- Internal --- - -_ATOMIC_KEYS = frozenset({"tags"}) - - -def _select(blueprint: Blueprint, section: Literal["inventory", "topology"], variant: str, scope: Scope) -> str | None: - """``None`` when the section is selected, else the skip reason. Raises for invalid requests.""" - names = blueprint.variant_names(section) - if not names: - if scope == section: - raise BlueprintValidationError( - ValidationIssue( - path=section, - message=f"scope='{section}' requested but the blueprint has no '{section}' section.", - code="section.missing", - ) - ) - if variant != "default": - raise BlueprintValidationError( - ValidationIssue( - path=section, - message=f"variant '{variant}' requested for the absent '{section}' section.", - code="section.missing", - ) - ) - return "section absent from blueprint" - if variant not in names: - raise BlueprintValidationError( - ValidationIssue( - path=section, - message=f"Unknown {section} variant '{variant}'. Available: {', '.join(names)}.", - code="variant.unknown", - ) - ) - if scope not in ("all", section): - return f"not in scope '{scope}'" - return None - - -def _raw_entries(blueprint: Blueprint, section: str) -> dict[str, dict[str, Any]]: - document = blueprint.model_dump(mode="python", by_alias=True, exclude_unset=True) - return document.get(section) or {} - - -def _merge( - base: dict[str, Any], patch: dict[str, Any], path: tuple[str, ...], provenance: Provenance, source: str -) -> dict[str, Any]: - merged = dict(base) - for key, value in patch.items(): - child = path + (key,) - current = merged.get(key) - if isinstance(value, dict) and isinstance(current, dict) and not _atomic(child): - merged[key] = _merge(current, value, child, provenance, source) - continue - merged[key] = copy.deepcopy(value) - prefix = ".".join(child) - for stale in [p for p in provenance if p == prefix or p.startswith(prefix + ".")]: - del provenance[stale] - provenance.update(_leaf_provenance(value, source, child)) - return merged - - -def _merge_processor(base: Any, patch: Any, provenance: Provenance, source: str) -> Any: - """``null`` disables; a different ``processor_type`` replaces the whole spec; else params merge.""" - replaces = ( - patch is None - or not isinstance(base, dict) - or not isinstance(patch, dict) - or patch.get("processor_type", base.get("processor_type")) != base.get("processor_type") - ) - if not replaces: - return _merge(base, patch, ("vertex_processor",), provenance, source) - for stale in [p for p in provenance if p == "vertex_processor" or p.startswith("vertex_processor.")]: - del provenance[stale] - provenance.update(_leaf_provenance(patch, source, ("vertex_processor",))) - return copy.deepcopy(patch) - - -def _atomic(path: tuple[str, ...]) -> bool: - """Naming expressions and tag specifications are replaced as a whole.""" - return (len(path) == 2 and path[0] == "naming") or path[-1] in _ATOMIC_KEYS or path[-1].endswith("_tags") - - -def _leaf_provenance(value: Any, source: str, path: tuple[str, ...] = ()) -> Provenance: - if isinstance(value, dict) and value and not (path and _atomic(path)): - result: Provenance = {} - for key, child in value.items(): - result.update(_leaf_provenance(child, source, path + (str(key),))) - return result - return {".".join(path): source} if path else {} - - -def _source_path( - entry_path: tuple[str, ...], relative: tuple[Any, ...], provenance: Provenance, key: str -) -> tuple[Any, ...]: - """Document path of a merged field: point at the entry the value actually came from.""" - source = provenance.get(key) or next( - (src for path, src in provenance.items() if path.startswith(key + ".") or key.startswith(path + ".")), None - ) - section = entry_path[0] - if source == "default": - return (section, "default") + relative - if source == "instance": - return ("device", "inventory_overrides") + relative - return entry_path + relative - - -def _validate_entry( - model: type[BaseModel], - data: dict[str, Any], - blueprint: Blueprint, - entry_path: tuple[str, ...], - provenance: Provenance, -) -> Any: - try: - return model.model_validate(data) - except ValidationError as exc: - issues = issues_from_pydantic(exc, data=data) - raise BlueprintValidationError( - [_relocate_relative(blueprint, entry_path, issue, provenance) for issue in issues] - ) from None - - -def _validate_custom_settings( - model: type[BaseModel], - custom: dict[str, Any], - blueprint: Blueprint, - entry_path: tuple[str, ...], - provenance: Provenance, - driver_id: str, -) -> dict[str, Any]: - allowed = set(model.model_fields) - {"driver_id"} - unknown = sorted(set(custom) - allowed) - if unknown: - raise BlueprintValidationError( - [ - _issue( - blueprint, - _source_path(entry_path, ("custom_settings", key), provenance, f"custom_settings.{key}"), - f"Unknown custom setting for driver '{driver_id}' (schema {SELECTED_SCHEMA_VERSION}). " - f"Allowed: {', '.join(sorted(allowed)) or 'none'}.", - "driver.unknown_setting", - ) - for key in unknown - ] - ) - try: - instance = model.model_validate(custom, strict=True, by_name=True, by_alias=False) - except ValidationError as exc: - issues = issues_from_pydantic(exc, data=custom) - raise BlueprintValidationError( - [ - _relocate( - blueprint, - _source_path(entry_path, ("custom_settings",), provenance, f"custom_settings.{issue.path}"), - issue.model_copy( - update={"message": f"{issue.message} (driver '{driver_id}', schema {SELECTED_SCHEMA_VERSION})"} - ), - ) - for issue in issues - ] - ) from None - return {key: getattr(instance, key) for key in custom} - - -def _relocate_relative( - blueprint: Blueprint, entry_path: tuple[str, ...], issue: ValidationIssue, provenance: Provenance -) -> ValidationIssue: - relative = tuple(issue.path.split(".")) if issue.path else () - path = _source_path(entry_path, relative, provenance, issue.path) - return _issue(blueprint, path, issue.message, issue.code) - - -def _relocate(blueprint: Blueprint, prefix: tuple[Any, ...], issue: ValidationIssue) -> ValidationIssue: - relative = tuple(issue.path.split(".")) if issue.path else () - return _issue(blueprint, prefix + relative, issue.message, issue.code) - - -def _issue(blueprint: Blueprint, path: tuple[Any, ...], message: str, code: str) -> ValidationIssue: - keys = tuple(int(part) if isinstance(part, str) and part.isdigit() else part for part in path) - source, line, column = None, None, None - for end in range(len(keys), -1, -1): - source, line, column = blueprint.location_of(keys[:end]) - if line is not None: - break - return ValidationIssue( - path=".".join(str(part) for part in path), - message=message, - code=code, - source=source, - line=line, - column=column, - ) - - -__all__ = [ - "ResolvedBlueprint", - "ResolvedInventory", - "ResolvedTopology", - "blueprint_digest", - "blueprint_json_schema", - "published_json_schema", - "resolve_blueprint", - "resolve_inventory", - "resolve_topology", - "validate_all_variants", -] diff --git a/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json b/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json deleted file mode 100644 index 97bba6c..0000000 --- a/src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json +++ /dev/null @@ -1,1159 +0,0 @@ -{ - "$defs": { - "BlueprintNaming": { - "additionalProperties": false, - "description": "The YAML form of naming entries (no Python renderers).", - "properties": { - "device_description": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Device Description" - }, - "device_label": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Device Label" - }, - "endpoint_description": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Endpoint Description" - }, - "endpoint_label": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Endpoint Label" - }, - "inventory_description": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Inventory Description" - }, - "inventory_label": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Inventory Label" - } - }, - "title": "BlueprintNaming", - "type": "object" - }, - "CatalogId": { - "additionalProperties": false, - "description": "Exact catalog id reference, e.g. ``{id: default}`` for the system SNMP configuration.", - "properties": { - "id": { - "minLength": 1, - "title": "Id", - "type": "string" - } - }, - "required": [ - "id" - ], - "title": "CatalogId", - "type": "object" - }, - "Coordinates": { - "additionalProperties": false, - "properties": { - "x": { - "title": "X", - "type": "number" - }, - "y": { - "title": "Y", - "type": "number" - } - }, - "required": [ - "x", - "y" - ], - "title": "Coordinates", - "type": "object" - }, - "DevicePatch": { - "additionalProperties": false, - "description": "Inspect device fields (device targets only). Device label/description come from naming.", - "properties": { - "coordinates": { - "anyOf": [ - { - "$ref": "#/$defs/Coordinates" - }, - { - "type": "null" - } - ], - "default": null - }, - "icon_size": { - "anyOf": [ - { - "enum": [ - "auto", - "large", - "medium", - "small" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Icon Size" - }, - "icon_type": { - "anyOf": [ - { - "enum": [ - "default", - "none", - "device", - "camera", - "monitor", - "encoder", - "decoder", - "audioMixer", - "videoMixer", - "processingDevice", - "transportStreamProcessor", - "mediaDevice", - "server", - "gateway", - "ipSwitchRouter", - "vlanCloud", - "videoAudioRouterMatrix", - "encoderDecoder" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Icon Type" - }, - "sdp_strategy": { - "anyOf": [ - { - "enum": [ - "always", - "once", - "video" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sdp Strategy" - }, - "site_id": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Site Id" - }, - "tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tags" - } - }, - "title": "DevicePatch", - "type": "object" - }, - "Field": { - "additionalProperties": false, - "description": "One naming fact: ``Field(\"endpoint.index\", format=\"02d\")`` or YAML ``{field: endpoint.index, format: 02d}``.\n\n``mapping`` translates the (stringified) value; ``prefix`` / ``suffix`` are only emitted when the\nvalue is present, so an absent optional field leaves no dangling separator text.", - "properties": { - "field": { - "title": "Field", - "type": "string" - }, - "format": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Format" - }, - "mapping": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Mapping" - }, - "prefix": { - "default": "", - "title": "Prefix", - "type": "string" - }, - "suffix": { - "default": "", - "title": "Suffix", - "type": "string" - } - }, - "required": [ - "field" - ], - "title": "Field", - "type": "object" - }, - "GenericSettings": { - "additionalProperties": false, - "description": "Generic connection settings (``cinfo.http``). ``http_auth`` is the server's raw\n``httpAuth`` mode code (``0`` disables HTTP authentication).", - "properties": { - "enable_https": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Enable Https" - }, - "http_auth": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Http Auth" - }, - "trust_all_certificates": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Trust All Certificates" - } - }, - "title": "GenericSettings", - "type": "object" - }, - "InventoryConfig": { - "additionalProperties": false, - "description": "One Inventory entry (``default`` or a named variant patch).", - "properties": { - "active": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Active" - }, - "custom_settings": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Custom Settings" - }, - "driver_id": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Driver Id" - }, - "generic_settings": { - "anyOf": [ - { - "$ref": "#/$defs/GenericSettings" - }, - { - "type": "null" - } - ], - "default": null - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - }, - { - "type": "number" - }, - { - "type": "boolean" - } - ] - }, - "propertyNames": { - "minLength": 1 - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Metadata" - }, - "naming": { - "anyOf": [ - { - "$ref": "#/$defs/BlueprintNaming" - }, - { - "type": "null" - } - ], - "default": null - }, - "snmp": { - "anyOf": [ - { - "$ref": "#/$defs/SnmpSelection" - }, - { - "type": "null" - } - ], - "default": null - } - }, - "title": "InventoryConfig", - "type": "object" - }, - "Join": { - "additionalProperties": false, - "description": "Join rendered parts with ``separator``. With ``skip_missing``, parts whose *optional* value is\nabsent are omitted; unknown fields, type errors, and failed mappings are still errors.", - "properties": { - "join": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/Text" - }, - { - "$ref": "#/$defs/Field" - }, - { - "$ref": "#/$defs/Join" - } - ] - }, - "title": "Join", - "type": "array" - }, - "separator": { - "default": "", - "title": "Separator", - "type": "string" - }, - "skip_missing": { - "default": false, - "title": "Skip Missing", - "type": "boolean" - } - }, - "required": [ - "join" - ], - "title": "Join", - "type": "object" - }, - "ModulePatch": { - "additionalProperties": false, - "description": "Module-local tag settings (module targets only).", - "properties": { - "tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tags" - } - }, - "title": "ModulePatch", - "type": "object" - }, - "SnmpSelection": { - "additionalProperties": false, - "description": "Global SNMP selection. ``configuration`` is an exact label (string) or ``{id: ...}``.", - "properties": { - "configuration": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "$ref": "#/$defs/CatalogId" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Configuration" - }, - "use_global_settings": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Use Global Settings" - } - }, - "title": "SnmpSelection", - "type": "object" - }, - "TagDelta": { - "additionalProperties": false, - "description": "Additive tag change that preserves unrelated local tags. Tag references are exact catalog\ntag ids (for example ``Format~~V_1080p50``); the engine never creates catalog entries.", - "properties": { - "add": { - "items": { - "minLength": 1, - "type": "string" - }, - "title": "Add", - "type": "array" - }, - "remove": { - "items": { - "minLength": 1, - "type": "string" - }, - "title": "Remove", - "type": "array" - } - }, - "title": "TagDelta", - "type": "object" - }, - "Text": { - "additionalProperties": false, - "description": "A literal text block: ``Text(\"-\")`` or YAML ``{text: \"-\"}``.", - "properties": { - "text": { - "title": "Text", - "type": "string" - } - }, - "required": [ - "text" - ], - "title": "Text", - "type": "object" - }, - "TopologyConfig": { - "additionalProperties": false, - "description": "One topology entry (``default`` or a named variant patch).", - "properties": { - "device": { - "anyOf": [ - { - "$ref": "#/$defs/DevicePatch" - }, - { - "type": "null" - } - ], - "default": null - }, - "ip_vertex_mapping": { - "anyOf": [ - { - "additionalProperties": { - "items": { - "minLength": 1, - "type": "string" - }, - "minItems": 1, - "type": "array" - }, - "propertyNames": { - "minLength": 1 - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Ip Vertex Mapping" - }, - "module": { - "anyOf": [ - { - "$ref": "#/$defs/ModulePatch" - }, - { - "type": "null" - } - ], - "default": null - }, - "naming": { - "anyOf": [ - { - "$ref": "#/$defs/BlueprintNaming" - }, - { - "type": "null" - } - ], - "default": null - }, - "vertex_processor": { - "anyOf": [ - { - "$ref": "#/$defs/VertexProcessorSpec" - }, - { - "type": "null" - } - ], - "default": null - }, - "vertices": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/VertexOverride" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Vertices" - } - }, - "title": "TopologyConfig", - "type": "object" - }, - "VertexOverride": { - "additionalProperties": false, - "description": "Explicit vertex edit. Select by exact ``vertex_id``, or by exact ``factory_label`` (supports\n``{module.position}``) optionally restricted by ``kind`` / ``direction``; must match exactly one vertex.", - "properties": { - "direction": { - "anyOf": [ - { - "enum": [ - "In", - "Out", - "Internal", - "Undecided" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Direction" - }, - "factory_label": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Factory Label" - }, - "fields": { - "$ref": "#/$defs/VertexPatch" - }, - "kind": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Kind" - }, - "vertex_id": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Vertex Id" - } - }, - "required": [ - "fields" - ], - "title": "VertexOverride", - "type": "object" - }, - "VertexPatch": { - "additionalProperties": false, - "description": "Supported vertex edits. ``\"\"`` is an explicit clear for label/description.\n\nKind-specific fields: ``sdp_support`` and the destination ports apply to codec vertices,\n``supports_static_igmp`` to IP vertices.", - "properties": { - "active": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Active" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Description" - }, - "label": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Label" - }, - "main_destination_port": { - "anyOf": [ - { - "maximum": 65535, - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Main Destination Port" - }, - "sdp_support": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sdp Support" - }, - "sips_mode": { - "anyOf": [ - { - "enum": [ - "NONE", - "SIPSAuto", - "SIPSDuplicate", - "SIPSMerge", - "SIPSSplit" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sips Mode" - }, - "spare_destination_port": { - "anyOf": [ - { - "maximum": 65535, - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Spare Destination Port" - }, - "supports_static_igmp": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Supports Static Igmp" - }, - "tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tags" - }, - "use_as_endpoint": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Use As Endpoint" - } - }, - "title": "VertexPatch", - "type": "object" - }, - "VertexProcessorSpec": { - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "processor_type": { - "const": "matrox.convertip.default" - } - }, - "required": [ - "processor_type" - ] - }, - "then": { - "properties": { - "params": { - "additionalProperties": false, - "description": "Parameters. Omitted tag parameters leave local tags unchanged (``[]`` clears them); an omitted\n``redundant_streams`` preserves the SIPS mode (``true`` \u2192 ``SIPSAuto``, ``false`` \u2192 ``NONE``).", - "properties": { - "allow_null_audio_label": { - "default": false, - "title": "Allow Null Audio Label", - "type": "boolean" - }, - "audio_receiver_tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Audio Receiver Tags" - }, - "audio_sender_tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Audio Sender Tags" - }, - "configure_control": { - "default": false, - "title": "Configure Control", - "type": "boolean" - }, - "mode": { - "default": "auto", - "enum": [ - "auto", - "tx", - "rx", - "four_split" - ], - "title": "Mode", - "type": "string" - }, - "redundant_streams": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Redundant Streams" - }, - "video_receiver_tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Video Receiver Tags" - }, - "video_sender_tags": { - "anyOf": [ - { - "items": { - "minLength": 1, - "type": "string" - }, - "type": "array" - }, - { - "$ref": "#/$defs/TagDelta" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Video Sender Tags" - } - }, - "title": "MatroxConvertIPParams", - "type": "object" - } - } - } - } - ], - "description": "Processor declaration. ``processor_type`` may be inherited from ``default`` by a variant;\nit is required on the resolved configuration.", - "properties": { - "params": { - "additionalProperties": true, - "title": "Params", - "type": "object" - }, - "processor_type": { - "anyOf": [ - { - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Processor Type" - } - }, - "title": "VertexProcessorSpec", - "type": "object" - } - }, - "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "additionalProperties": false, - "description": "A reusable, type-level configuration with ``default`` entries and optional named variants.\n\nLoad with :meth:`load` (file), :meth:`from_yaml` (text/bytes), or :meth:`from_dict` (mapping).\nLoading never connects to a server. A named variant is a patch over its section's ``default``.", - "properties": { - "inventory": { - "anyOf": [ - { - "patternProperties": { - "^[A-Za-z0-9][A-Za-z0-9_.-]*$": { - "$ref": "#/$defs/InventoryConfig" - } - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Inventory" - }, - "schema_version": { - "const": 1, - "title": "Schema Version", - "type": "integer" - }, - "topology": { - "anyOf": [ - { - "patternProperties": { - "^[A-Za-z0-9][A-Za-z0-9_.-]*$": { - "$ref": "#/$defs/TopologyConfig" - } - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Topology" - } - }, - "required": [ - "schema_version" - ], - "title": "VideoIPath Automation Tool blueprint (schema_version 1)", - "type": "object" -} diff --git a/src/videoipath_automation_tool/blueprints/__init__.py b/src/videoipath_automation_tool/provisioning/__init__.py similarity index 50% rename from src/videoipath_automation_tool/blueprints/__init__.py rename to src/videoipath_automation_tool/provisioning/__init__.py index ed337cc..0062de0 100644 --- a/src/videoipath_automation_tool/blueprints/__init__.py +++ b/src/videoipath_automation_tool/provisioning/__init__.py @@ -1,7 +1,9 @@ -"""Blueprint-driven Inventory and Inspect configuration. +"""Declarative provisioning of VideoIPath devices through the existing SDK apps. -A standalone feature: construct :class:`BlueprintEngine` with an app (``BlueprintEngine(app)``), -describe a device with :class:`BlueprintDevice`, then pass a YAML file path or :class:`Blueprint` to +Blueprints are reusable configuration templates within this provisioning layer. + +A standalone feature: construct :class:`ProvisioningEngine` with an app (``ProvisioningEngine(app)``), +describe a device with :class:`ProvisioningDevice`, then pass a YAML file path or :class:`Blueprint` to ``engine.plan(...)`` (read-only preview) and ``plan.apply()``, or ``engine.apply(...)`` in one call. ``VideoIPathApp`` neither imports nor exposes this package. @@ -10,45 +12,54 @@ from __future__ import annotations -from videoipath_automation_tool.blueprints.engine import BlueprintApp, BlueprintEngine, BlueprintPlan -from videoipath_automation_tool.blueprints.errors import ( - BlueprintApplyError, - BlueprintCapabilityError, - BlueprintConflictError, - BlueprintError, - BlueprintTargetError, - BlueprintValidationError, +from videoipath_automation_tool.provisioning.engine import ProvisioningApp, ProvisioningEngine, ProvisioningPlan +from videoipath_automation_tool.provisioning.errors import ( + InventoryNotReadyError, ProcessorInputError, + ProvisioningApplyError, + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningError, + ProvisioningTargetError, + ProvisioningValidationError, TopologyNotReadyError, ValidationIssue, ) -from videoipath_automation_tool.blueprints.models import ( +from videoipath_automation_tool.provisioning.inputs import InputDefinition, InputReference, LiteralValue +from videoipath_automation_tool.provisioning.models import ( AlternativeAddress, ApplyOptions, ApplyResult, Blueprint, - BlueprintDevice, + BlueprintConfiguration, CatalogId, + ConnectionState, Coordinates, Credentials, DevicePatch, DeviceTarget, Diagnostic, + EdgePatch, EndpointIdentity, FieldChange, InterfaceBinding, InventorySettings, ModulePatch, ModuleTarget, + PeerEndpoint, PhaseResult, PlannedOperation, PlannedPhase, + PortBinding, + PortSelector, ProcessorResult, + ProvisioningConnection, + ProvisioningDevice, TagDelta, VertexEdit, VertexPatch, ) -from videoipath_automation_tool.blueprints.naming import ( +from videoipath_automation_tool.provisioning.naming import ( DEFAULT_NAMING, Field, Join, @@ -57,7 +68,7 @@ NamingScheme, Text, ) -from videoipath_automation_tool.blueprints.processors import ( +from videoipath_automation_tool.provisioning.processors import ( DeviceRecord, DriverContext, ModuleRecord, @@ -68,8 +79,8 @@ VertexProcessor, VertexRecord, ) -from videoipath_automation_tool.blueprints.processors.matrox import MatroxConvertIPParams, MatroxConvertIPProcessor -from videoipath_automation_tool.blueprints.resolution import published_json_schema +from videoipath_automation_tool.provisioning.processors.matrox import MatroxConvertIPParams, MatroxConvertIPProcessor +from videoipath_automation_tool.provisioning.resolution import published_json_schema __all__ = [ "DEFAULT_NAMING", @@ -77,17 +88,9 @@ "ApplyOptions", "ApplyResult", "Blueprint", - "BlueprintApp", - "BlueprintApplyError", - "BlueprintCapabilityError", - "BlueprintConflictError", - "BlueprintDevice", - "BlueprintEngine", - "BlueprintError", - "BlueprintPlan", - "BlueprintTargetError", - "BlueprintValidationError", + "BlueprintConfiguration", "CatalogId", + "ConnectionState", "Coordinates", "Credentials", "DevicePatch", @@ -95,12 +98,17 @@ "DeviceTarget", "Diagnostic", "DriverContext", + "EdgePatch", "EndpointIdentity", "Field", "FieldChange", + "InputDefinition", + "InputReference", "InterfaceBinding", + "InventoryNotReadyError", "InventorySettings", "Join", + "LiteralValue", "MatroxConvertIPParams", "MatroxConvertIPProcessor", "ModulePatch", @@ -109,14 +117,28 @@ "NameContext", "NameRenderer", "NamingScheme", + "PeerEndpoint", "PhaseResult", "PlannedOperation", "PlannedPhase", + "PortBinding", "PortRecord", + "PortSelector", "ProcessingContext", "ProcessorInputError", "ProcessorRegistry", "ProcessorResult", + "ProvisioningApp", + "ProvisioningApplyError", + "ProvisioningCapabilityError", + "ProvisioningConflictError", + "ProvisioningConnection", + "ProvisioningDevice", + "ProvisioningEngine", + "ProvisioningError", + "ProvisioningPlan", + "ProvisioningTargetError", + "ProvisioningValidationError", "SourceFacts", "TagDelta", "Text", diff --git a/src/videoipath_automation_tool/provisioning/connections.py b/src/videoipath_automation_tool/provisioning/connections.py new file mode 100644 index 0000000..ef4494b --- /dev/null +++ b/src/videoipath_automation_tool/provisioning/connections.py @@ -0,0 +1,279 @@ +"""Pure port resolution and managed edge diffs. All server I/O stays in InspectGateway.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from pydantic import BaseModel, ConfigDict, Field + +from videoipath_automation_tool.apps.inspect.model.actions import InspectApiEdgeForm +from videoipath_automation_tool.apps.inspect.model.common import CONFLICT_PRIORITY_TO_INT +from videoipath_automation_tool.provisioning.errors import ( + ProvisioningTargetError, + ProvisioningValidationError, + TopologyNotReadyError, +) +from videoipath_automation_tool.provisioning.models import ( + ConnectionState, + EdgePatch, + FieldChange, + PlannedOperation, + PortBinding, + PortSelector, + ProvisioningConnection, + TagDelta, +) + +if TYPE_CHECKING: + from videoipath_automation_tool.provisioning.inspect import ScopeData + + +class EdgeWork(BaseModel): + model_config = ConfigDict(frozen=True) + + from_vertex: str + to_vertex: str + fields: EdgePatch + baseline: InspectApiEdgeForm | None = None + intents: dict[str, Any] = Field(default_factory=dict) + expected: dict[str, Any] = Field(default_factory=dict) + operation: PlannedOperation | None = None + + +class ConnectionWork(BaseModel): + model_config = ConfigDict(frozen=True) + + states: list[ConnectionState] = Field(default_factory=list) + edges: dict[str, EdgeWork] = Field(default_factory=dict) + + @property + def pending(self) -> bool: + return any(state.status == "deferred" for state in self.states) + + @property + def operations(self) -> list[PlannedOperation]: + return [edge.operation for edge in self.edges.values() if edge.operation is not None] + + def touched_keys(self) -> set[tuple[str, str]]: + keys = {("edge", edge_id) for edge_id in self.edges} + for state in self.states: + for binding in (state.local, state.peer): + if binding is None: + continue + keys.add(("device", binding.device_id)) + if binding.module_id is not None: + keys.add(("module", binding.module_id)) + keys.update(("vertex", vid) for vid in (binding.in_vertex_id, binding.out_vertex_id) if vid) + return keys + + +def resolve_port(scope: ScopeData, candidates: list[PortSelector], *, key: str) -> PortBinding: + """First matching candidate wins; ambiguity never falls through to another candidate.""" + if not scope.ports: + raise TopologyNotReadyError(f"No ports discovered for '{scope.module_id or scope.device_id}'.") + for candidate in candidates: + ports = [ + port + for port in scope.ports + if ( + port.id == candidate.port_id + if candidate.port_id is not None + else port.factory_label == candidate.factory_label + ) + and ( + candidate.kind is None or any(v.port_id == port.id and v.kind == candidate.kind for v in scope.vertices) + ) + ] + if len(ports) > 1: + raise ProvisioningTargetError( + f"Port '{key}' matches multiple ports in '{scope.module_id or scope.device_id}'." + ) + if not ports: + continue + port = ports[0] + vertices = [ + vertex + for vertex in scope.vertices + if vertex.port_id == port.id and (candidate.kind is None or vertex.kind == candidate.kind) + ] + directions: dict[str, str | None] = {} + for direction in ("In", "Out"): + matches = [v.id for v in vertices if v.vertex_type == direction] + if len(matches) > 1: + raise ProvisioningTargetError(f"Port '{port.id}' has multiple '{direction}' vertices.") + directions[direction] = matches[0] if matches else None + if not any(directions.values()): + raise ProvisioningTargetError(f"Port '{port.id}' has no directed In/Out vertex.") + return PortBinding( + key=key, + candidate=candidate.port_id or candidate.factory_label or "", + port_id=port.id, + device_id=scope.device_id, + module_id=port.module_id, + in_vertex_id=directions["In"], + out_vertex_id=directions["Out"], + ) + raise ProvisioningTargetError(f"Port '{key}' has no match in '{scope.module_id or scope.device_id}'.") + + +def resolve_connections( + connections: list[ProvisioningConnection], + scope: ScopeData, + peers: dict[int, ScopeData | None], + bindings: list[PortBinding], +) -> ConnectionWork: + """Resolve concrete endpoints; an absent peer stays deferred until a new plan.""" + by_key = {binding.key: binding for binding in bindings} + states: list[ConnectionState] = [] + edges: dict[str, EdgeWork] = {} + for index, connection in enumerate(connections): + if connection.peer.target.device_id == scope.device_id: + raise ProvisioningTargetError("Connections must target another device; internal topology is driver-owned.") + if isinstance(connection.local, str): + if connection.local not in by_key: + raise ProvisioningValidationError(f"Unknown local port mapping '{connection.local}'.") + local = by_key[connection.local] + else: + local = resolve_port(scope, [connection.local], key=f"connections.{index}.local") + peer_scope = peers[index] + if peer_scope is None or not peer_scope.ports: + states.append( + ConnectionState( + index=index, + connection=connection, + local=local, + status="deferred", + reason="Peer device, module or ports are not discovered in topology; create a new plan when available.", + ) + ) + continue + peer = resolve_port(peer_scope, [connection.peer.port], key=f"connections.{index}.peer") + pairs = _directed_pairs(local, peer, connection.direction) + edge_ids: list[str] = [] + for from_vertex, to_vertex in pairs: + edge_id = f"{from_vertex}::{to_vertex}" + edge_ids.append(edge_id) + fields = connection.fields + if edge_id in edges: + managed = edges[edge_id].fields.managed() + for name, value in fields.managed().items(): + if name in managed and managed[name] != value: + raise ProvisioningValidationError(f"Conflicting '{name}' requirements for edge '{edge_id}'.") + managed[name] = value + fields = EdgePatch.model_validate(managed) + edges[edge_id] = EdgeWork(from_vertex=from_vertex, to_vertex=to_vertex, fields=fields) + states.append( + ConnectionState( + index=index, + connection=connection, + local=local, + peer=peer, + edge_ids=edge_ids, + status="planned", + ) + ) + return ConnectionWork(states=states, edges=edges) + + +def compare_edges(work: ConnectionWork, current: dict[str, InspectApiEdgeForm]) -> ConnectionWork: + """Compare only managed fields while keeping full baselines for conflict checks.""" + edges: dict[str, EdgeWork] = {} + for edge_id, edge in work.edges.items(): + baseline = current.get(edge_id) + form = baseline or InspectApiEdgeForm(fromId=edge.from_vertex, toId=edge.to_vertex) + expected: dict[str, Any] = {"fromId": edge.from_vertex, "toId": edge.to_vertex} + intents: dict[str, Any] = {} + changes: list[FieldChange] = [] + for name, value in edge.fields.managed().items(): + path = _EDGE_PATHS.get(name, name) + before = edge_value(form, path) + if name == "tags": + if isinstance(value, TagDelta): + value = [tag for tag in form.tags if tag not in value.remove] + [ + tag for tag in dict.fromkeys(value.add) if tag not in form.tags + ] + else: + value = list(dict.fromkeys(value)) + elif name == "conflict_priority" and isinstance(value, str): + value = CONFLICT_PRIORITY_TO_INT[value] + expected[path] = value + if not same_edge_value(path, before, value): + intents[path] = value + changes.append(FieldChange(field=name, before=before, after=value, source="instance")) + operation = None + if baseline is None or intents: + if baseline is None: + changes = [ + FieldChange(field="from_vertex", after=edge.from_vertex, source="instance"), + FieldChange(field="to_vertex", after=edge.to_vertex, source="instance"), + *changes, + ] + operation = PlannedOperation( + action="create" if baseline is None else "update", + entity_kind="edge", + entity_id=edge_id, + changes=changes, + ) + edges[edge_id] = edge.model_copy( + update={ + "baseline": baseline.model_copy(deep=True) if baseline is not None else None, + "intents": intents, + "expected": expected, + "operation": operation, + } + ) + states = [ + state + if state.status == "deferred" + else state.model_copy( + update={ + "status": "planned" if any(edges[eid].operation is not None for eid in state.edge_ids) else "no_change", + } + ) + for state in work.states + ] + return ConnectionWork(states=states, edges=edges) + + +def edge_value(form: InspectApiEdgeForm, path: str) -> Any: + value: Any = form + for part in path.split("."): + value = value.get(part) if isinstance(value, dict) else getattr(value, part, None) + if path == "conflictPri": + if isinstance(value, str) and value in CONFLICT_PRIORITY_TO_INT: + return CONFLICT_PRIORITY_TO_INT[value] + if isinstance(value, str) and value.isdigit(): + return int(value) + return value + + +def same_edge_value(path: str, before: Any, after: Any) -> bool: + return set(before or []) == set(after or []) if path == "tags" else before == after + + +def _directed_pairs(local: PortBinding, peer: PortBinding, direction: str) -> list[tuple[str, str]]: + outgoing = (local.out_vertex_id, peer.in_vertex_id) + incoming = (peer.out_vertex_id, local.in_vertex_id) + required = ( + [outgoing, incoming] + if direction in ("auto", "bidirectional") + else [outgoing if direction == "outgoing" else incoming] + ) + pairs = [(source, target) for source, target in required if source is not None and target is not None] + if not pairs or (direction != "auto" and len(pairs) != len(required)): + raise ProvisioningTargetError( + f"Ports '{local.port_id}' and '{peer.port_id}' cannot satisfy direction '{direction}'." + ) + return pairs + + +_EDGE_PATHS = { + "label": "descriptor.label", + "description": "descriptor.desc", + "redundancy_mode": "redundancyMode", + "conflict_priority": "conflictPri", + "include_formats": "includeFormats", + "exclude_formats": "excludeFormats", + "bandwidth_weight_factor": "weightFactors.bandwidth.weight", + "weight_per_service": "weightFactors.service.weight", +} diff --git a/src/videoipath_automation_tool/blueprints/engine.py b/src/videoipath_automation_tool/provisioning/engine.py similarity index 67% rename from src/videoipath_automation_tool/blueprints/engine.py rename to src/videoipath_automation_tool/provisioning/engine.py index 59d899f..51b1501 100644 --- a/src/videoipath_automation_tool/blueprints/engine.py +++ b/src/videoipath_automation_tool/provisioning/engine.py @@ -1,6 +1,6 @@ -"""The standalone :class:`BlueprintEngine`: configuration, read-only planning, and execution. +"""The standalone :class:`ProvisioningEngine`: configuration, read-only planning, and execution. -``engine.plan(...)`` reads VideoIPath state and returns an immutable :class:`BlueprintPlan`; it never +``engine.plan(...)`` reads VideoIPath state and returns an immutable :class:`ProvisioningPlan`; it never writes, synchronizes, stages snapshot edits, or creates catalog entries. ``plan.apply()`` executes exactly that plan through the shared executor, checking the captured baselines first. ``engine.apply(...)`` is ``engine.plan(...).apply()``. ``dry_run=True`` runs the same execution path — @@ -11,11 +11,11 @@ plan again after the driver has rediscovered the device. Creating a record still waits for discovery during apply, because a new device has no previous topology. -Execution phases (absent or unchanged phases are skipped): Inventory → discovery readiness → -topology membership/synchronization → topology commit (one ``InspectTransaction``) → module tags +Execution phases (absent or unchanged phases are skipped): Inventory → Inventory reachability → +topology membership/synchronization → required topology data → topology commit (one ``InspectTransaction``) → module tags (separate RPCs) → verification. Inventory and Inspect writes are separate operations; there is no transaction spanning the whole deployment and no automatic rollback. Failures raise -:class:`BlueprintApplyError` carrying an :class:`ApplyResult` with every known id and phase outcome. +:class:`ProvisioningApplyError` carrying an :class:`ApplyResult` with every known id and phase outcome. """ from __future__ import annotations @@ -23,32 +23,35 @@ import contextlib import logging import time -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from os import PathLike from typing import Any, Protocol -from pydantic import BaseModel, ConfigDict, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field, JsonValue, PrivateAttr from videoipath_automation_tool.apps.inspect.errors import InspectCommitConflictError, InspectCommitError +from videoipath_automation_tool.apps.inspect.model.actions import InspectApiLookupSyncInfoItem from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice -from videoipath_automation_tool.blueprints.errors import ( - BlueprintApplyError, - BlueprintCapabilityError, - BlueprintConflictError, - BlueprintError, - BlueprintTargetError, - BlueprintValidationError, +from videoipath_automation_tool.provisioning.errors import ( + InventoryNotReadyError, + ProvisioningApplyError, + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningError, + ProvisioningTargetError, + ProvisioningValidationError, TopologyNotReadyError, ) -from videoipath_automation_tool.blueprints.inspect import ( +from videoipath_automation_tool.provisioning.inspect import ( InspectGateway, + ScopeData, TopologyWork, compute_topology_work, name_context, verify_topology, ) -from videoipath_automation_tool.blueprints.inventory import ( +from videoipath_automation_tool.provisioning.inventory import ( InventoryGateway, InventoryWork, build_candidate, @@ -58,11 +61,11 @@ recheck_conflicts, verify, ) -from videoipath_automation_tool.blueprints.models import ( +from videoipath_automation_tool.provisioning.models import ( ApplyOptions, ApplyResult, Blueprint, - BlueprintDevice, + ConnectionState, DeviceTarget, Diagnostic, InterfaceBinding, @@ -71,24 +74,33 @@ PhaseResult, PlannedOperation, PlannedPhase, + PortBinding, + ProvisioningDevice, Scope, TopologyTarget, ) -from videoipath_automation_tool.blueprints.naming import DEFAULT_NAMING, NamingScheme, render_name -from videoipath_automation_tool.blueprints.processors import ( +from videoipath_automation_tool.provisioning.naming import ( + DEFAULT_NAMING, + INVENTORY_NAMING_ENTRIES, + TOPOLOGY_NAMING_ENTRIES, + NamingScheme, + render_name, + validate_naming_inputs, +) +from videoipath_automation_tool.provisioning.processors import ( DriverContext, ProcessorRegistry, SourceFacts, VertexProcessor, ) -from videoipath_automation_tool.blueprints.resolution import ResolvedBlueprint, resolve_blueprint +from videoipath_automation_tool.provisioning.resolution import ResolvedBlueprint, resolve_blueprint from videoipath_automation_tool.validators.device_id import validate_device_id # Sentinel for "argument not supplied" (distinct from an explicit None). _UNSET: Any = object() -class BlueprintApp(Protocol): +class ProvisioningApp(Protocol): """The app interface the engine needs; :class:`VideoIPathApp` satisfies it unchanged.""" @property @@ -100,21 +112,21 @@ def inspect(self) -> Any: """The Inspect app (``InspectApp``).""" -class BlueprintPlan(BaseModel): +class ProvisioningPlan(BaseModel): """A reviewed, immutable preview of intended changes. ``apply()`` executes exactly this plan. - ``fully_resolved`` is ``False`` when topology work depends on earlier phases. Inventory creation, + ``fully_resolved`` is ``False`` when topology work depends on earlier phases or peers are missing. Inventory creation, topology membership, and pending synchronization are materialized during ``apply()`` from the captured configuration. A topology-affecting Inventory update is not: ``apply()`` stops after - Inventory and the result asks for a new plan. + Inventory and the result asks for a new plan. Missing peers stay deferred until a new plan; + the remaining local configuration and resolvable connections can still be applied. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) source_key: str scope: Scope - inventory_variant: str | None = None - topology_variant: str | None = None + variants: tuple[str, ...] = () blueprint_digest: str driver_id: str | None = None driver_schema_version: str | None = None @@ -123,6 +135,8 @@ class BlueprintPlan(BaseModel): topology_target: TopologyTarget | None = None phases: list[PlannedPhase] interface_bindings: list[InterfaceBinding] + port_bindings: list[PortBinding] = Field(default_factory=list) + connections: list[ConnectionState] = Field(default_factory=list) diagnostics: list[Diagnostic] skipped_sections: dict[str, str] fully_resolved: bool @@ -154,10 +168,8 @@ def apply(self, *, dry_run: bool = False) -> ApplyResult: def summary(self) -> str: """Human-readable, redacted summary with before/after values and unresolved work.""" header = f"Blueprint plan for '{self.source_key}' (scope={self.scope}" - if self.inventory_variant: - header += f", inventory={self.inventory_variant}" - if self.topology_variant: - header += f", topology={self.topology_variant}" + if self.variants: + header += f", variants={list(self.variants)}" lines = [header + f", digest={self.blueprint_digest[:12]})"] for phase in self.phases: line = f"{phase.name} [{phase.status}]" @@ -170,10 +182,22 @@ def summary(self) -> str: lines.append(f" ~ {change.field}: {change.before!r} -> {change.after!r} ({change.source})") for binding in self.interface_bindings: lines.append(f"interface {binding.key} -> port {binding.port_id} ({binding.candidate})") + for binding in self.port_bindings: + lines.append(f"port {binding.key} -> {binding.port_id} ({binding.candidate})") + for connection in self.connections: + lines.append( + f"connection {connection.index} [{connection.status}]: " + + (connection.reason or ", ".join(connection.edge_ids)) + ) for diagnostic in self.diagnostics: lines.append(f"[{diagnostic.level}] {diagnostic.code}: {diagnostic.message}") if not self.fully_resolved: - if self._captured.topology_requires_replan: + if ( + any(connection.status == "deferred" for connection in self.connections) + and not self._captured.topology_deferred + ): + lines.append("Connections remain open: create a new plan when the peer topology is available.") + elif self._captured.topology_requires_replan: lines.append( "Not fully resolved: apply stops after Inventory; replan once the driver has rediscovered " "the device." @@ -187,13 +211,13 @@ def summary(self) -> str: def __repr__(self) -> str: statuses = ", ".join(f"{phase.name}={phase.status}" for phase in self.phases) - return f"BlueprintPlan(source_key={self.source_key!r}, {statuses}, fully_resolved={self.fully_resolved})" + return f"ProvisioningPlan(source_key={self.source_key!r}, {statuses}, fully_resolved={self.fully_resolved})" __str__ = __repr__ -class BlueprintEngine: - """Standalone blueprint engine bound to one injected app for its lifetime. +class ProvisioningEngine: + """Standalone provisioning engine bound to one injected app for its lifetime. Only ``app`` is required. ``processors`` (id → class) are added to the built-ins, ``options`` and ``naming`` set engine-wide defaults; each has a post-initialization equivalent @@ -204,7 +228,7 @@ class BlueprintEngine: def __init__( self, - app: BlueprintApp, + app: ProvisioningApp, *, processors: Mapping[str, type[VertexProcessor[Any]]] | None = None, options: ApplyOptions | None = None, @@ -212,7 +236,7 @@ def __init__( logger: logging.Logger | None = None, ) -> None: self._app = app - self._logger = logger or logging.getLogger("videoipath_automation_tool_blueprints") + self._logger = logger or logging.getLogger("videoipath_automation_tool_provisioning") self._registry = ProcessorRegistry(processors) self._options = ApplyOptions() self._naming: NamingScheme | None = None @@ -226,7 +250,7 @@ def __init__( # --- Configuration --- @property - def app(self) -> BlueprintApp: + def app(self) -> ProvisioningApp: return self._app @property @@ -264,15 +288,22 @@ def configure( if naming is not _UNSET: self._naming = naming.model_copy() if naming is not None else None - def validate(self, blueprint: Blueprint | str | PathLike[str]) -> None: - """Validate every variant of ``blueprint`` against this engine's processors (offline). + def validate( + self, + blueprint: Blueprint | str | PathLike[str], + *, + inputs: Mapping[str, JsonValue] | None = None, + variants: Sequence[str] | None = None, + ) -> None: + """Validate defaults and individual variants, or an explicit combination, offline. Accepts a Blueprint instance or a YAML file path (string or path-like object). + ``inputs`` binds declared values; ``variants`` selects one ordered combination. Raises: - BlueprintValidationError: listing all issues, including unregistered processors. + ProvisioningValidationError: listing all issues, including unregistered processors. """ - self._load_blueprint(blueprint).validate_full(self._registry) + self._load_blueprint(blueprint).validate_full(self._registry, inputs=inputs, variants=variants) def json_schema(self) -> dict[str, Any]: """Blueprint JSON Schema for this engine: ``processor_type`` is limited to the registered @@ -283,23 +314,23 @@ def json_schema(self) -> dict[str, Any]: def plan( self, - device: BlueprintDevice, + device: ProvisioningDevice, blueprint: Blueprint | str | PathLike[str], *, scope: Scope = "all", - inventory_variant: str = "default", - topology_variant: str = "default", + inputs: Mapping[str, JsonValue] | None = None, + variants: Sequence[str] = (), naming: NamingScheme | None = None, options: ApplyOptions | None = None, - ) -> BlueprintPlan: + ) -> ProvisioningPlan: """Load/resolve the blueprint, read current state, and compute changes. Read-only. ``blueprint`` accepts a Blueprint instance or a YAML file path (string or path-like object). Files are loaded once during planning; applying the plan never re-reads them. Use ``Blueprint.from_yaml(text)`` for raw YAML strings. """ - if not isinstance(device, BlueprintDevice): - raise TypeError("device must be a BlueprintDevice.") + if not isinstance(device, ProvisioningDevice): + raise TypeError("device must be a ProvisioningDevice.") if naming is not None and not isinstance(naming, NamingScheme): raise TypeError("naming must be a NamingScheme.") if options is not None and not isinstance(options, ApplyOptions): @@ -312,11 +343,22 @@ def plan( resolved = resolve_blueprint( blueprint, scope=scope, - inventory_variant=inventory_variant, - topology_variant=topology_variant, + inputs=inputs, + variants=variants, registry=registry, overrides=device.inventory_overrides, ) + if device.connections and scope != "inventory": + if resolved.topology is None: + raise ProvisioningValidationError( + "Connections require a topology section in the selected configuration." + ) + keys = set(resolved.topology.config.ip_vertex_mapping or {}) | set( + resolved.topology.config.port_mapping or {} + ) + for connection in device.connections: + if isinstance(connection.local, str) and connection.local not in keys: + raise ProvisioningValidationError(f"Unknown local port mapping '{connection.local}'.") captured = _Captured( device=device, resolved=resolved, @@ -324,16 +366,20 @@ def plan( options=effective_options, registry=registry, ) + active_naming = (INVENTORY_NAMING_ENTRIES if resolved.inventory else frozenset()) | ( + TOPOLOGY_NAMING_ENTRIES if resolved.topology else frozenset() + ) + validate_naming_inputs(captured.naming, blueprint.inputs, resolved.inputs, entries=active_naming) return _Planner(self._app, captured, scope, self._executor()).build() def apply( self, - device: BlueprintDevice, + device: ProvisioningDevice, blueprint: Blueprint | str | PathLike[str], *, scope: Scope = "all", - inventory_variant: str = "default", - topology_variant: str = "default", + inputs: Mapping[str, JsonValue] | None = None, + variants: Sequence[str] = (), naming: NamingScheme | None = None, options: ApplyOptions | None = None, dry_run: bool = False, @@ -346,14 +392,14 @@ def apply( device, blueprint, scope=scope, - inventory_variant=inventory_variant, - topology_variant=topology_variant, + inputs=inputs, + variants=variants, naming=naming, options=options, ).apply(dry_run=dry_run) def __repr__(self) -> str: - return f"BlueprintEngine(processors={self._registry.ids()}, options={self._options!r})" + return f"ProvisioningEngine(processors={self._registry.ids()}, options={self._options!r})" # --- Internal --- @@ -377,7 +423,7 @@ class _Captured(BaseModel): model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - device: BlueprintDevice + device: ProvisioningDevice resolved: ResolvedBlueprint naming: NamingScheme options: ApplyOptions @@ -387,6 +433,7 @@ class _Captured(BaseModel): topology_work: TopologyWork | None = None topology_deferred: bool = False topology_requires_replan: bool = False + connection_peers: dict[int, ScopeData | None] = Field(default_factory=dict) @property def source(self) -> SourceFacts: @@ -404,19 +451,29 @@ def source(self) -> SourceFacts: class _Planner: - def __init__(self, app: BlueprintApp, captured: _Captured, scope: Scope, executor: _Executor) -> None: + def __init__(self, app: ProvisioningApp, captured: _Captured, scope: Scope, executor: _Executor) -> None: self._app = app self._captured = captured self._scope = scope self._executor = executor self._inventory_gateway: InventoryGateway | None = None - def build(self) -> BlueprintPlan: + def build(self) -> ProvisioningPlan: captured = self._captured resolved = captured.resolved phases: list[PlannedPhase] = [] diagnostics: list[Diagnostic] = [] + if resolved.topology is not None and captured.device.connections: + captured = captured.model_copy( + update={ + "connection_peers": InspectGateway(self._app.inspect).read_connection_peers( + captured.device.connections + ) + } + ) + self._captured = captured + inventory_work, current = self._plan_inventory(phases) target, deferred_reason, topology_work, topology_requires_replan = self._plan_topology( inventory_work, current, phases, diagnostics @@ -431,11 +488,10 @@ def build(self) -> BlueprintPlan: "topology_requires_replan": topology_requires_replan, } ) - plan = BlueprintPlan( + plan = ProvisioningPlan( source_key=captured.device.key, scope=self._scope, - inventory_variant=resolved.inventory.variant if resolved.inventory else None, - topology_variant=resolved.topology.variant if resolved.topology else None, + variants=resolved.variants, blueprint_digest=resolved.digest, driver_id=resolved.inventory.driver_id if resolved.inventory else None, driver_schema_version=resolved.inventory.schema_version if resolved.inventory else None, @@ -444,9 +500,17 @@ def build(self) -> BlueprintPlan: topology_target=target, phases=phases, interface_bindings=list(topology_work.bindings) if topology_work else [], + port_bindings=list(topology_work.port_bindings) if topology_work else [], + connections=[state.model_copy(deep=True) for state in topology_work.connections.states] + if topology_work + else [ + ConnectionState(index=index, connection=connection, status="deferred", reason=deferred_reason) + for index, connection in enumerate(captured.device.connections) + if resolved.topology is not None + ], diagnostics=diagnostics + (list(topology_work.diagnostics) if topology_work else []), skipped_sections=dict(resolved.skipped), - fully_resolved=deferred_reason is None, + fully_resolved=deferred_reason is None and not (topology_work and topology_work.connections.pending), ) plan._captured = captured plan._executor = self._executor @@ -458,7 +522,7 @@ def _plan_inventory(self, phases: list[PlannedPhase]) -> tuple[InventoryWork | N phases.append(PlannedPhase(name="inventory", status="skipped", reason=resolved.skipped.get("inventory"))) return None, None naming = self._captured.naming - context = name_context(self._captured.source) + context = name_context(self._captured.source, inputs=self._captured.resolved.inputs) label = render_name("inventory_label", naming.inventory_label, context) if naming.inventory_label else None description = ( render_name("inventory_description", naming.inventory_description, context) @@ -503,7 +567,7 @@ def _plan_topology( target = DeviceTarget(device_id=device.inventory_id) creating = inventory_work is not None and inventory_work.action == "create" if target is None and not creating: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Topology configuration for '{device.key}' needs an explicit 'topology' target or an 'inventory_id'." ) @@ -522,18 +586,33 @@ def _plan_topology( else: reason = self._membership_reason(target, diagnostics) + work: TopologyWork | None = None + if reason is None: + assert target is not None + try: + work = self._materialize(target, current) + except TopologyNotReadyError as exc: + reason = str(exc) + if reason is not None: - phases.extend(PlannedPhase(name=name, status="deferred", reason=reason) for name in _TOPOLOGY_PHASES) + for name in _TOPOLOGY_PHASES: + skip = _inventory_readiness_skip(captured, target) if name == "inventory_readiness" else None + phases.append(PlannedPhase(name=name, status="skipped" if skip else "deferred", reason=skip or reason)) return target, reason, None, requires_replan - assert target is not None - work = self._materialize(target, current) - phases.append(PlannedPhase(name="discovery", status="skipped", reason="topology already present")) + assert work is not None + phases.append(PlannedPhase(name="inventory_readiness", status="skipped", reason="topology already ready")) phases.append(PlannedPhase(name="topology_sync", status="no_change")) + phases.append(PlannedPhase(name="discovery", status="skipped", reason="topology already ready")) phases.append( PlannedPhase( name="topology", - status="planned" if work.has_topology_writes else "no_change", + status="planned" + if work.has_topology_writes + else ("deferred" if work.connections.pending else "no_change"), + reason="Peer connections remain open; create a new plan when available." + if work.connections.pending + else None, operations=work.topology_operations, ) ) @@ -552,7 +631,7 @@ def _membership_reason(self, target: TopologyTarget, diagnostics: list[Diagnosti gateway = InspectGateway(self._app.inspect) if not gateway.in_topology(target.device_id): if sync == "none" or isinstance(target, ModuleTarget): - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Inspect device '{target.device_id}' is not in the topology" + ( " (sync='none')." @@ -577,7 +656,7 @@ def _membership_reason(self, target: TopologyTarget, diagnostics: list[Diagnosti ) return None if sync == "add_only" and (info.update or info.remove): - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( f"Synchronizing '{target.device_id}' requires updates/removals; sync='add_only' is insufficient " "(use sync='reconcile' deliberately, or synchronize through the Inspect API)." ) @@ -593,20 +672,25 @@ def _inventory(self) -> InventoryGateway: def _materialize_topology( - app: BlueprintApp, + app: ProvisioningApp, captured: _Captured, target: TopologyTarget, inventory: Callable[[], InventoryGateway], current: InventoryDevice | None, + *, + gateway: InspectGateway | None = None, ) -> TopologyWork: """Collect a fresh scoped context and compute exact topology edits from the captured configuration.""" assert captured.resolved.topology is not None - scope = InspectGateway(app.inspect).read_scope(target) + gateway = gateway or InspectGateway(app.inspect) + scope = gateway.read_scope(target) + gateway.check_deadline() own = _driver_context(current) if current is not None else None owner = ( own if own is not None and own.inventory_id == target.device_id else _owner_context(inventory, target.device_id) ) - return compute_topology_work( + gateway.check_deadline() + work = compute_topology_work( scope=scope, resolved=captured.resolved.topology, naming=captured.naming, @@ -615,6 +699,9 @@ def _materialize_topology( inventory=own, allow_label_collisions=captured.options.naming_collisions == "allow", ) + return gateway.plan_connections( + work, scope, captured.device.connections, captured.connection_peers, captured.device.module_position + ) # --- Internal: execution --- @@ -624,19 +711,24 @@ class _Executor: """Executes plans through the bound app; uses only the plan's captured settings.""" def __init__( - self, app: BlueprintApp, *, clock: Callable[[], float], sleep: Callable[[float], None], logger: logging.Logger + self, + app: ProvisioningApp, + *, + clock: Callable[[], float], + sleep: Callable[[float], None], + logger: logging.Logger, ) -> None: self._app = app self._clock = clock self._sleep = sleep self._logger = logger - def execute(self, plan: BlueprintPlan, *, dry_run: bool = False) -> ApplyResult: + def execute(self, plan: ProvisioningPlan, *, dry_run: bool = False) -> ApplyResult: return _Execution(self, plan, dry_run=dry_run).run() class _Execution: - def __init__(self, executor: _Executor, plan: BlueprintPlan, *, dry_run: bool) -> None: + def __init__(self, executor: _Executor, plan: ProvisioningPlan, *, dry_run: bool) -> None: self._executor = executor self._app = executor._app self._plan = plan @@ -644,6 +736,12 @@ def __init__(self, executor: _Executor, plan: BlueprintPlan, *, dry_run: bool) - self._dry_run = dry_run self._would_write = False self._result = ApplyResult(source_key=plan.source_key, inventory_id=plan.inventory_id, dry_run=dry_run) + self._result.connections = [ + state.model_copy(deep=True, update={"status": "not_run"}) + if state.status == "planned" + else state.model_copy(deep=True) + for state in plan.connections + ] self._phases: dict[str, PhaseResult] = { phase.name: PhaseResult( name=phase.name, @@ -661,6 +759,8 @@ def __init__(self, executor: _Executor, plan: BlueprintPlan, *, dry_run: bool) - self._inventory_gateway: InventoryGateway | None = None self._current_inventory: InventoryDevice | None = None self._topology_readback_done = False + self._topology_add_requested = False + self._requested_sync: InspectApiLookupSyncInfoItem | None = None def run(self) -> ApplyResult: try: @@ -714,6 +814,8 @@ def _run_topology(self) -> None: if captured.topology_requires_replan: reason = self._plan.phase("topology").reason if self._plan.phase("topology") else None for name in _TOPOLOGY_PHASES: + if self._phases[name].status == "skipped": + continue phase = self._enter(name) phase.status = "deferred" phase.message = reason @@ -724,6 +826,8 @@ def _run_topology(self) -> None: if captured.topology_deferred and self._dry_run: reason = self._plan.phase("topology").reason if self._plan.phase("topology") else None for name in _TOPOLOGY_PHASES: + if self._phases[name].status == "skipped": + continue phase = self._enter(name) phase.status = "deferred" phase.message = f"not run in a dry run: {reason}" @@ -732,30 +836,45 @@ def _run_topology(self) -> None: target = captured.topology_target if target is None: if self._result.inventory_id is None: - raise BlueprintTargetError("No topology target: the Inventory record id is unknown.") + raise ProvisioningTargetError("No topology target: the Inventory record id is unknown.") target = DeviceTarget(device_id=self._result.inventory_id) self._result.topology_device_id = target.device_id self._result.module_id = target.module_id if isinstance(target, ModuleTarget) else None gateway = InspectGateway(self._app.inspect) if captured.topology_deferred: - work = self._discover_and_materialize(gateway, target) + self._wait_for_inventory(target) + work = self._discover_and_materialize(target) self._result.materialized = True else: assert captured.topology_work is not None work = captured.topology_work self._enter("topology") if gateway.read_scope(target).fingerprint != work.fingerprint: - raise BlueprintConflictError( + raise ProvisioningConflictError( f"The topology of {_target_label(target)} changed since planning; create a new plan." ) + self._enter("topology") self._result.interface_bindings = list(work.bindings) + self._result.port_bindings = list(work.port_bindings) + self._result.connections = [ + state.model_copy(deep=True, update={"status": "not_run"}) + if state.status == "planned" and not self._dry_run + else state.model_copy(deep=True) + for state in work.connections.states + ] + self._result.replan_required = work.connections.pending + if work.connections.pending and self._dry_run: + self._would_write = True self._result.diagnostics.extend(work.diagnostics) + gateway.check_peers(captured.device.connections, captured.connection_peers) + gateway.check_edges(work.connections) + overlap = gateway.staged_edit_keys() & work.touched_keys() if overlap: entities = ", ".join(f"{kind} '{entity_id}'" for kind, entity_id in sorted(overlap)) - raise BlueprintConflictError( + raise ProvisioningConflictError( f"Pending uncommitted Inspect edits overlap this plan ({entities}); commit or discard them first." ) @@ -768,13 +887,20 @@ def _run_topology(self) -> None: phase.status = "completed" phase.operations = list(work.topology_operations) self._wrote.append("topology") + self._result.connections = [ + state.model_copy(update={"status": "completed"}) if state.status == "not_run" else state + for state in self._result.connections + ] else: - phase.status = "no_change" + phase.status = "deferred" if work.connections.pending else "no_change" + if work.connections.pending: + phase.message = "Peer connections remain open; create a new plan when available." self._run_module_tags(gateway, work) if "topology" in self._wrote or "module_tags" in self._wrote: try: self._mismatches.extend(verify_topology(gateway.read_scope(target), work)) + self._mismatches.extend(gateway.verify_edges(work.connections)) except Exception as exc: # noqa: BLE001 - verification never turns an applied write into a failure self._verification_errors.append(f"topology read-back failed: {type(exc).__name__}: {exc}") finally: @@ -791,7 +917,7 @@ def _run_module_tags(self, gateway: InspectGateway, work: TopologyWork) -> None: return current = gateway.module_local_tags(work.device_id, work.module_id) if work.module_tag_baseline is not None and set(current or ()) != set(work.module_tag_baseline): - raise BlueprintConflictError( + raise ProvisioningConflictError( f"Local tags of module '{work.module_id}' changed since planning; the tag phase was not applied." ) if self._dry_run: @@ -810,58 +936,122 @@ def _run_module_tags(self, gateway: InspectGateway, work: TopologyWork) -> None: gateway.refresh_device(work.device_id) phase.status = "completed" - def _discover_and_materialize(self, gateway: InspectGateway, target: TopologyTarget) -> TopologyWork: - options = self._captured.options - discovery = self._enter("discovery") - deadline = self._executor._clock() + options.discovery_timeout - sync_phase = self._phases["topology_sync"] + def _wait_for_inventory(self, target: TopologyTarget) -> None: + phase = self._enter("inventory_readiness") + skip = _inventory_readiness_skip(self._captured, target) + if skip: + phase.status = "skipped" + phase.message = skip + return + inventory_id = self._result.inventory_id or self._captured.device.inventory_id or target.device_id + wait = self._readiness_wait("inventory_readiness", inventory_id, self._captured.options.inventory_ready_timeout) + gateway = self._inventory() + try: + wait.check() + record = self._current_inventory + if record is None or record.device_id != inventory_id: + record = gateway.read(inventory_id) + while True: + wait.check() + status = gateway.read_status(record) + wait.condition = "status unavailable" if status is None else f"reachable={status.reachable}" + wait.check() + if status is not None and status.reachable: + phase.status = "completed" + phase.message = f"Inventory device '{inventory_id}' is reachable" + return + wait.pause(wait.condition) + except _ReadinessExpired as exc: + raise InventoryNotReadyError(str(exc)) from None + + def _discover_and_materialize(self, target: TopologyTarget) -> TopologyWork: + wait = self._readiness_wait("topology_sync", target.device_id, self._captured.options.topology_ready_timeout) + gateway = InspectGateway(self._app.inspect, check_deadline=wait.check) + sync_phase = self._enter("topology_sync") sync_phase.status = "no_change" - while True: - try: - self._ensure_membership(gateway, target) - self._enter("topology") - work = _materialize_topology( - self._app, self._captured, target, self._inventory, self._current_inventory - ) - discovery.status = "completed" - return work - except TopologyNotReadyError as exc: - if self._unknown: - raise - remaining = deadline - self._executor._clock() - if remaining <= 0: - self._current = "discovery" - raise TopologyNotReadyError( - f"Topology not ready after {options.discovery_timeout:g}s: {exc}" - ) from exc - self._executor._sleep(min(options.poll_interval, remaining)) - - def _ensure_membership(self, gateway: InspectGateway, target: TopologyTarget) -> None: + try: + while True: + wait.check() + try: + wait.stage = "topology_sync" + self._ensure_membership(gateway, target, wait) + wait.check() + wait.stage = "discovery" + wait.condition = "waiting for required topology data" + discovery = self._enter("discovery") + work = _materialize_topology( + self._app, self._captured, target, self._inventory, self._current_inventory, gateway=gateway + ) + wait.check() + discovery.status = "completed" + discovery.message = f"Required topology data for '{target.device_id}' is ready" + return work + except TopologyNotReadyError as exc: + if self._unknown: + raise + wait.pause(str(exc)) + except _ReadinessExpired as exc: + raise TopologyNotReadyError(str(exc)) from None + + def _readiness_wait(self, stage: PhaseName, device_id: str, timeout: float) -> _ReadinessWait: + return _ReadinessWait( + stage=stage, + device_id=device_id, + timeout=timeout, + poll_interval=self._captured.options.poll_interval, + clock=self._executor._clock, + sleep=self._executor._sleep, + ) + + def _ensure_membership(self, gateway: InspectGateway, target: TopologyTarget, wait: _ReadinessWait) -> None: sync = self._captured.options.sync phase = self._enter("topology_sync") if isinstance(target, ModuleTarget): # Never add or synchronize a whole parent device to reach a module target. if not gateway.in_topology(target.device_id): - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Parent device '{target.device_id}' of module '{target.module_id}' is not in the topology." ) return if not gateway.in_topology(target.device_id): + wait.condition = "device is not yet in the topology" if sync == "none": - raise BlueprintTargetError(f"Inspect device '{target.device_id}' is not in the topology (sync='none').") + raise ProvisioningTargetError( + f"Inspect device '{target.device_id}' is not in the topology (sync='none')." + ) + if self._topology_add_requested: + raise TopologyNotReadyError("Topology addition accepted; waiting for the device to appear.") with self._write(): gateway.add_to_topology(target.device_id) + self._topology_add_requested = True self._record_sync(phase, "add_to_topology", target.device_id) + wait.condition = "Topology addition accepted; waiting for the device to appear." + if not gateway.in_topology(target.device_id): + raise TopologyNotReadyError(wait.condition) + wait.condition = "topology membership confirmed; synchronization status not yet available" info = gateway.sync_info(target.device_id) if not _sync_pending(info) or sync == "none": + self._requested_sync = None return if sync == "add_only" and (info.update or info.remove): - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( f"Synchronizing '{target.device_id}' requires updates/removals; sync='add_only' is insufficient." ) + previous = self._requested_sync + if previous is not None and (info.add, info.update, info.remove) == ( + previous.add, + previous.update, + previous.remove, + ): + raise TopologyNotReadyError("Topology synchronization is still pending.") with self._write(): gateway.sync(target.device_id, add_only=sync == "add_only") self._record_sync(phase, "sync", target.device_id) + wait.condition = "Topology synchronization accepted; waiting for completion." + self._requested_sync = info.model_copy(deep=True) + if _sync_pending(gateway.sync_info(target.device_id)): + raise TopologyNotReadyError("Topology synchronization is still pending.") + self._requested_sync = None def _record_sync(self, phase: PhaseResult, action: str, device_id: str) -> None: phase.operations.append(PlannedOperation(action=action, entity_kind="device", entity_id=device_id)) # type: ignore[arg-type] @@ -897,6 +1087,13 @@ def _fail(self, exc: Exception) -> None: phase = self._phases.setdefault(name, PhaseResult(name=name, status="not_run")) # type: ignore[arg-type] phase.status = "unknown" if self._unknown else "failed" phase.message = f"{type(exc).__name__}: {exc}" + if name == "topology": + self._result.connections = [ + state.model_copy(update={"status": "unknown" if self._unknown else "failed", "reason": phase.message}) + if state.status == "not_run" + else state + for state in self._result.connections + ] if self._unknown: self._result.status = "unknown" elif self._wrote: @@ -926,9 +1123,9 @@ def _finish(self) -> None: error = getattr(self, "_error", None) if error is not None: self._executor._logger.warning( - "Blueprint apply for '%s' ended %s: %s", result.source_key, result.status, error + "Provisioning apply for '%s' ended %s: %s", result.source_key, result.status, error ) - raise BlueprintApplyError(result, f"{type(error).__name__}: {error}") from error + raise ProvisioningApplyError(result, f"{type(error).__name__}: {error}") from error if self._dry_run: result.verification_detail = "dry run: nothing was written" result.status = "planned" if self._would_write else "no_change" @@ -936,7 +1133,7 @@ def _finish(self) -> None: result.status = "partial" else: result.status = "succeeded" if self._wrote else "no_change" - self._executor._logger.info("Blueprint apply for '%s' %s.", result.source_key, result.status) + self._executor._logger.info("Provisioning apply for '%s' %s.", result.source_key, result.status) def _inventory(self) -> InventoryGateway: if self._inventory_gateway is None: @@ -946,13 +1143,64 @@ def _inventory(self) -> InventoryGateway: # --- Internal helpers --- -_TOPOLOGY_PHASES: tuple[PhaseName, ...] = ("discovery", "topology_sync", "topology", "module_tags") -_PHASE_ORDER: tuple[str, ...] = ("inventory", "discovery", "topology_sync", "topology", "module_tags", "verification") +_TOPOLOGY_PHASES: tuple[PhaseName, ...] = ( + "inventory_readiness", + "topology_sync", + "discovery", + "topology", + "module_tags", +) +_PHASE_ORDER: tuple[str, ...] = ("inventory", *_TOPOLOGY_PHASES, "verification") + + +class _ReadinessExpired(ProvisioningError): + """Internal deadline signal; kept distinct from a retryable not-ready observation.""" + + +class _ReadinessWait(BaseModel): + """A fixed monotonic budget, including read time, shared by one readiness stage.""" + + stage: PhaseName + device_id: str + timeout: float + poll_interval: float + clock: Callable[[], float] + sleep: Callable[[float], None] + condition: str = "no status observed yet" + _deadline: float = PrivateAttr() + + def model_post_init(self, context: Any) -> None: + self._deadline = self.clock() + self.timeout + + def check(self) -> None: + if self.clock() >= self._deadline: + raise _ReadinessExpired( + f"{self.stage} for device '{self.device_id}' not ready after {self.timeout:g}s: {self.condition}" + ) + + def pause(self, condition: str) -> None: + self.condition = condition + self.check() + self.sleep(min(self.poll_interval, max(0.0, self._deadline - self.clock()))) + + +def _inventory_readiness_skip(captured: _Captured, target: TopologyTarget | None) -> str | None: + if not captured.options.require_reachable: + return "Inventory reachability check disabled (require_reachable=False)" + if target is not None and target.device_id.startswith("virtual."): + return "virtual topology target" + if ( + isinstance(target, ModuleTarget) + and captured.device.inventory_id is None + and captured.resolved.inventory is None + ): + return "module-only topology target without its own Inventory binding" + return None def _known_rejection(exc: Exception, *, create: bool) -> bool: """Whether a write error proves the server did not apply the change.""" - if isinstance(exc, (InspectCommitError, InspectCommitConflictError, BlueprintError)): + if isinstance(exc, (InspectCommitError, InspectCommitConflictError, ProvisioningError)): return True if isinstance(exc, InventoryWriteNotAppliedError): # A create is only known to be rejected when the add itself failed: the follow-up update @@ -987,10 +1235,10 @@ def _owner_context(inventory: Callable[[], InventoryGateway], device_id: str) -> return None # e.g. virtual devices have no Inventory record try: return _driver_context(inventory().read(device_id)) - except (BlueprintTargetError, BlueprintValidationError): + except (ProvisioningTargetError, ProvisioningValidationError): return None -BlueprintPlan.model_rebuild() +ProvisioningPlan.model_rebuild() -__all__ = ["BlueprintApp", "BlueprintEngine", "BlueprintPlan"] +__all__ = ["ProvisioningApp", "ProvisioningEngine", "ProvisioningPlan"] diff --git a/src/videoipath_automation_tool/blueprints/errors.py b/src/videoipath_automation_tool/provisioning/errors.py similarity index 69% rename from src/videoipath_automation_tool/blueprints/errors.py rename to src/videoipath_automation_tool/provisioning/errors.py index 0fcdad7..ea08724 100644 --- a/src/videoipath_automation_tool/blueprints/errors.py +++ b/src/videoipath_automation_tool/provisioning/errors.py @@ -1,6 +1,6 @@ -"""Typed exceptions for blueprint loading, planning, and application. +"""Typed exceptions for provisioning: blueprint loading, planning, and application. -Every error derives from :class:`BlueprintError`. Validation problems carry structured +Every error derives from :class:`ProvisioningError`. Validation problems carry structured :class:`ValidationIssue` records (document path, optional YAML location, issue code) instead of raw values, so messages never echo credentials or arbitrary server payloads. """ @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict if TYPE_CHECKING: - from videoipath_automation_tool.blueprints.models import ApplyResult + from videoipath_automation_tool.provisioning.models import ApplyResult class ValidationIssue(BaseModel): @@ -42,11 +42,11 @@ def __str__(self) -> str: return "\n".join(lines) -class BlueprintError(Exception): - """Base class for all blueprint errors.""" +class ProvisioningError(Exception): + """Base class for all provisioning errors.""" -class BlueprintValidationError(BlueprintError): +class ProvisioningValidationError(ProvisioningError): """A blueprint, device description, naming rule, or processor parameter set is invalid.""" def __init__(self, issues: Iterable[ValidationIssue] | ValidationIssue | str, *, code: str = "invalid") -> None: @@ -55,38 +55,42 @@ def __init__(self, issues: Iterable[ValidationIssue] | ValidationIssue | str, *, elif isinstance(issues, ValidationIssue): issues = [issues] self.issues: list[ValidationIssue] = list(issues) - super().__init__("\n\n".join(str(issue) for issue in self.issues) or "Blueprint validation failed.") + super().__init__("\n\n".join(str(issue) for issue in self.issues) or "Provisioning validation failed.") @property def codes(self) -> list[str]: return [issue.code for issue in self.issues] -class BlueprintTargetError(BlueprintError): +class ProvisioningTargetError(ProvisioningError): """A binding is missing or ambiguous, or a target lies outside the requested scope.""" -class ProcessorInputError(BlueprintError): +class ProcessorInputError(ProvisioningError): """A processor cannot interpret the observed topology (unsupported or incomplete layout).""" -class TopologyNotReadyError(BlueprintError): +class InventoryNotReadyError(ProvisioningError): + """Inventory did not report the device as reachable within its readiness deadline.""" + + +class TopologyNotReadyError(ProvisioningError): """The discovered topology is still incomplete; waiting for discovery may resolve it. - While topology work is deferred, ``apply()`` retries this error until ``discovery_timeout``. + While topology work is deferred, ``apply()`` retries this error until ``topology_ready_timeout``. Every other error fails the attempt immediately. """ -class BlueprintCapabilityError(BlueprintError): +class ProvisioningCapabilityError(ProvisioningError): """A requested operation is not supported by the adapter or the server.""" -class BlueprintConflictError(BlueprintError): +class ProvisioningConflictError(ProvisioningError): """A reviewed plan or a managed baseline changed since it was captured; replan.""" -class BlueprintApplyError(BlueprintError): +class ProvisioningApplyError(ProvisioningError): """Execution failed, partially succeeded, or ended with an unknown outcome. ``result`` is the structured :class:`ApplyResult` describing every completed, failed, and @@ -96,17 +100,18 @@ class BlueprintApplyError(BlueprintError): def __init__(self, result: ApplyResult, message: str) -> None: self.result = result - super().__init__(f"Blueprint apply {result.status} for '{result.source_key}': {message}") + super().__init__(f"Provisioning apply {result.status} for '{result.source_key}': {message}") __all__ = [ - "BlueprintApplyError", - "BlueprintCapabilityError", - "BlueprintConflictError", - "BlueprintError", - "BlueprintTargetError", - "BlueprintValidationError", + "InventoryNotReadyError", "ProcessorInputError", + "ProvisioningApplyError", + "ProvisioningCapabilityError", + "ProvisioningConflictError", + "ProvisioningError", + "ProvisioningTargetError", + "ProvisioningValidationError", "TopologyNotReadyError", "ValidationIssue", ] diff --git a/src/videoipath_automation_tool/provisioning/inputs.py b/src/videoipath_automation_tool/provisioning/inputs.py new file mode 100644 index 0000000..b6ad2fc --- /dev/null +++ b/src/videoipath_automation_tool/provisioning/inputs.py @@ -0,0 +1,163 @@ +"""Strict, source-independent input declarations and typed value references.""" + +from __future__ import annotations + +import copy +import math +from collections.abc import Mapping +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, StringConstraints, model_validator + +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue + +InputName = Annotated[str, StringConstraints(pattern=r"^[A-Za-z_][A-Za-z0-9_-]*$")] + + +class InputReference(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, frozen=True, validate_by_name=True, validate_by_alias=True) + + name: InputName = Field(alias="$input") + + +class LiteralValue(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, frozen=True, validate_by_name=True, validate_by_alias=True) + + value: JsonValue = Field(alias="$literal") + + +class InputDefinition(BaseModel): + """A small JSON Schema vocabulary; defaults are values, never expressions.""" + + model_config = ConfigDict(extra="forbid", strict=True, frozen=True, validate_by_name=True, validate_by_alias=True) + + type: Literal["string", "integer", "number", "boolean", "array", "object", "null"] + description: str | None = None + default: JsonValue = None + enum: Annotated[list[JsonValue], Field(min_length=1)] | None = None + items: InputDefinition | None = None + properties: dict[str, InputDefinition] | None = None + required: list[str] | None = None + additional_properties: bool = Field(default=False, alias="additionalProperties") + + @model_validator(mode="after") + def _definition(self) -> InputDefinition: + supplied = self.model_fields_set + if "items" in supplied and self.type != "array": + raise ValueError("items is only valid for array inputs") + if supplied & {"properties", "required", "additional_properties"} and self.type != "object": + raise ValueError("properties, required and additionalProperties are only valid for object inputs") + if self.required is not None: + if len(set(self.required)) != len(self.required): + raise ValueError("required contains duplicate property names") + if set(self.required) - set(self.properties or {}): + raise ValueError("required must name declared properties") + for value in self.enum or []: + if value_issues(self, value, (), check_enum=False): + raise ValueError("enum entries must conform to the declared input type and structure") + if "default" in supplied: + issues = value_issues(self, self.default, ()) + if issues: + raise ValueError("invalid input default: " + "; ".join(issue.message for issue in issues)) + return self + + +def bind_inputs( + definitions: Mapping[str, InputDefinition], supplied: Mapping[str, JsonValue] | None +) -> dict[str, JsonValue]: + """Validate supplied values and copy defaults. Requiredness is checked at each use site.""" + if supplied is not None and not isinstance(supplied, Mapping): + raise TypeError("inputs must be a mapping or None.") + values = { + name: copy.deepcopy(spec.default) for name, spec in definitions.items() if "default" in spec.model_fields_set + } + issues: list[ValidationIssue] = [] + for name, value in (supplied or {}).items(): + if name not in definitions: + issues.append(ValidationIssue(path=f"inputs.{name}", message="Unknown input.", code="input.unknown")) + else: + issues.extend(value_issues(definitions[name], value, ("inputs", name))) + values[name] = copy.deepcopy(value) + if issues: + raise ProvisioningValidationError(issues) + return values + + +def value_issues( + definition: InputDefinition, value: Any, path: tuple[str | int, ...], *, check_enum: bool = True +) -> list[ValidationIssue]: + """Validate without coercion or including raw values in diagnostics.""" + kind = definition.type + matches = { + "string": isinstance(value, str), + "integer": type(value) is int, + "number": type(value) in (int, float) and (not isinstance(value, float) or math.isfinite(value)), + "boolean": type(value) is bool, + "array": isinstance(value, list), + "object": isinstance(value, dict) and all(isinstance(key, str) for key in value), + "null": value is None, + } + if not matches[kind]: + return [_issue(path, f"Expected input type '{kind}'; received a different type.", "input.type")] + issues: list[ValidationIssue] = [] + if check_enum and definition.enum is not None and not any(_equal(value, entry) for entry in definition.enum): + issues.append(_issue(path, "Input value is not in the declared enum.", "input.enum")) + if kind == "array": + for index, item in enumerate(value): + if definition.items is not None: + issues.extend(value_issues(definition.items, item, path + (index,))) + elif not _is_json(item): + issues.append(_issue(path + (index,), "Expected a JSON value.", "input.type")) + if kind == "object": + properties = definition.properties or {} + for name in definition.required or []: + if name not in value: + issues.append(_issue(path + (name,), "Required input property is missing.", "input.missing")) + for name, item in value.items(): + if name in properties: + issues.extend(value_issues(properties[name], item, path + (name,))) + elif not definition.additional_properties: + issues.append(_issue(path + (name,), "Unknown input property.", "input.unknown_property")) + elif not _is_json(item): + issues.append(_issue(path + (name,), "Expected a JSON value.", "input.type")) + return issues + + +def is_expression(value: Any) -> bool: + return isinstance(value, dict) and bool(value.keys() & {"$input", "$literal"}) + + +def contains_expression(value: Any) -> bool: + if isinstance(value, (InputReference, LiteralValue)): + return True + if isinstance(value, dict): + return is_expression(value) or any(contains_expression(child) for child in value.values()) + if isinstance(value, list): + return any(contains_expression(child) for child in value) + return False + + +def _issue(path: tuple[str | int, ...], message: str, code: str) -> ValidationIssue: + return ValidationIssue(path=".".join(map(str, path)), message=message, code=code) + + +def _equal(left: JsonValue, right: JsonValue) -> bool: + if type(left) is bool or type(right) is bool: + return type(left) is type(right) and left == right + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(_equal(a, b) for a, b in zip(left, right, strict=True)) + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all(_equal(left[key], right[key]) for key in left) + return left == right + + +def _is_json(value: Any) -> bool: + if value is None or type(value) in (str, bool, int): + return True + if type(value) is float: + return math.isfinite(value) + if isinstance(value, list): + return all(_is_json(item) for item in value) + if isinstance(value, dict): + return all(isinstance(key, str) and _is_json(item) for key, item in value.items()) + return False diff --git a/src/videoipath_automation_tool/blueprints/inspect.py b/src/videoipath_automation_tool/provisioning/inspect.py similarity index 76% rename from src/videoipath_automation_tool/blueprints/inspect.py rename to src/videoipath_automation_tool/provisioning/inspect.py index 37918d0..b13a64a 100644 --- a/src/videoipath_automation_tool/blueprints/inspect.py +++ b/src/videoipath_automation_tool/provisioning/inspect.py @@ -15,26 +15,38 @@ import json import re from collections import defaultdict +from collections.abc import Callable from typing import Any -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, JsonValue +from videoipath_automation_tool.apps.inspect.errors import InspectEntityNotFoundError +from videoipath_automation_tool.apps.inspect.model.actions import InspectApiEdgeForm from videoipath_automation_tool.apps.inspect.model.collector import ( InspectApiDoubleVertexInfo, InspectApiNodeStatusItem, InspectApiSingleVertexInfo, ) from videoipath_automation_tool.apps.inspect.model.tags import module_resource_id -from videoipath_automation_tool.blueprints.errors import ( - BlueprintCapabilityError, - BlueprintError, - BlueprintTargetError, - BlueprintValidationError, +from videoipath_automation_tool.provisioning.connections import ( + ConnectionWork, + compare_edges, + edge_value, + resolve_connections, + resolve_port, + same_edge_value, +) +from videoipath_automation_tool.provisioning.errors import ( ProcessorInputError, + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningError, + ProvisioningTargetError, + ProvisioningValidationError, TopologyNotReadyError, ValidationIssue, ) -from videoipath_automation_tool.blueprints.models import ( +from videoipath_automation_tool.provisioning.models import ( Coordinates, Diagnostic, EndpointIdentity, @@ -42,12 +54,15 @@ InterfaceBinding, ModuleTarget, PlannedOperation, + PortBinding, + PortSelector, ProcessorResult, + ProvisioningConnection, TagDelta, TopologyTarget, ) -from videoipath_automation_tool.blueprints.naming import NameContext, NamingScheme, render_name -from videoipath_automation_tool.blueprints.processors import ( +from videoipath_automation_tool.provisioning.naming import NameContext, NamingScheme, render_name +from videoipath_automation_tool.provisioning.processors import ( DeviceRecord, DriverContext, ModuleRecord, @@ -56,7 +71,7 @@ SourceFacts, VertexRecord, ) -from videoipath_automation_tool.blueprints.resolution import ResolvedTopology +from videoipath_automation_tool.provisioning.resolution import ResolvedTopology class ScopeData(BaseModel): @@ -114,13 +129,15 @@ class TopologyWork(BaseModel): expected_vertices: dict[str, dict[str, Any]] = Field(default_factory=dict) expected_module_tags: list[str] | None = None bindings: list[InterfaceBinding] = Field(default_factory=list) + port_bindings: list[PortBinding] = Field(default_factory=list) + connections: ConnectionWork = Field(default_factory=ConnectionWork) diagnostics: list[Diagnostic] = Field(default_factory=list) topology_operations: list[PlannedOperation] = Field(default_factory=list) module_operations: list[PlannedOperation] = Field(default_factory=list) @property def has_topology_writes(self) -> bool: - return bool(self.device_intents or self.vertex_intents) + return bool(self.device_intents or self.vertex_intents or self.connections.operations) @property def has_module_writes(self) -> bool: @@ -132,24 +149,33 @@ def touched_keys(self) -> set[tuple[str, str]]: keys.add(("device", self.device_id)) if self.has_module_writes and self.module_id is not None: keys.add(("module", self.module_id)) + keys.update(self.connections.touched_keys()) return keys class InspectGateway: """The only place that talks to ``app.inspect`` (public methods plus one internal API handle).""" - def __init__(self, inspect_app: Any) -> None: + def __init__(self, inspect_app: Any, *, check_deadline: Callable[[], None] | None = None) -> None: self._app = inspect_app + self._check_deadline = check_deadline + + def check_deadline(self) -> None: + """Prevent another discovery request after the rollout deadline.""" + if self._check_deadline is not None: + self._check_deadline() @property def _api(self) -> Any: + self.check_deadline() return self._app._inspect_api # --- Reads --- def in_topology(self, device_id: str) -> bool: if self._detail(device_id) is not None: - return True + # Collector also lists Inventory-only devices, before addDevices creates the edit form. + return self._device_form(device_id, required=False) is not None return device_id.startswith("virtual.") and self._device_form(device_id, required=False) is not None def sync_info(self, device_id: str) -> Any | None: @@ -158,12 +184,13 @@ def sync_info(self, device_id: str) -> Any | None: A failed lookup is an error. Treating it as "nothing pending" would let apply commit topology edits while a synchronization that updates or removes elements is still outstanding. """ + self.check_deadline() try: return self._app.get_sync_info([device_id]).get(device_id) - except BlueprintError: + except ProvisioningError: raise except Exception as exc: - raise BlueprintError( + raise ProvisioningError( f"Could not read synchronization status for '{device_id}': {type(exc).__name__}: {exc}" ) from exc @@ -173,12 +200,12 @@ def read_scope(self, target: TopologyTarget) -> ScopeData: node = self._detail(device_id) form = self._device_form(device_id, required=node is not None) if node is None and form is None: - raise BlueprintTargetError(f"Inspect device '{device_id}' is not in the topology.") + raise ProvisioningTargetError(f"Inspect device '{device_id}' is not in the topology.") modules = _modules_of(node) if module_id is not None and module_id not in modules: known = ", ".join(sorted(modules)[:10]) or "none" - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Module '{module_id}' does not belong to Inspect device '{device_id}' (modules: {known})." ) @@ -196,6 +223,27 @@ def read_scope(self, target: TopologyTarget) -> ScopeData: ) ports.append((record, sides)) + missing_labels = [vid for port, sides in ports if not port.factory_label for vid, _ in sides] + if missing_labels: + # Collector labels are absent on supported servers. Use the same immutable driver + # graph labels as InspectPort.factory_label, never the editable descriptor label. + factory_labels = self._api.get_ngraph_factory_labels(*missing_labels) + ports = [ + ( + port.model_copy( + update={ + "factory_label": next( + (factory_labels[vid] for vid, _ in sides if factory_labels.get(vid)), None + ) + } + ) + if not port.factory_label + else port, + sides, + ) + for port, sides in ports + ] + in_scope = [(p, sides) for p, sides in ports if module_id is None or p.module_id == module_id] scope_ids = [vid for _, sides in in_scope for vid, _ in sides] external_ids = [ @@ -230,7 +278,7 @@ def module_local_tags(self, device_id: str, module_id: str) -> tuple[str, ...] | node = self._detail(device_id) module = _modules_of(node).get(module_id) if module is None: - raise BlueprintTargetError(f"Module '{module_id}' of '{device_id}' is no longer in the topology.") + raise ProvisioningTargetError(f"Module '{module_id}' of '{device_id}' is no longer in the topology.") return _local_tags(module) def staged_edit_keys(self) -> set[tuple[str, str]]: @@ -240,15 +288,111 @@ def staged_edit_keys(self) -> set[tuple[str, str]]: return set() return {(kind, entity_id) for kind, entity_id, intents in snapshot.iter_staged_edits() if intents} + def read_peer_scope(self, target: TopologyTarget) -> ScopeData | None: + """Missing discovery is pending; transport and lookup failures propagate.""" + node = self._detail(target.device_id) + if node is None: + return None + if self._device_form(target.device_id, required=False) is None: + return None + if isinstance(target, ModuleTarget) and target.module_id not in _modules_of(node): + return None + scope = self.read_scope(target) + return scope if scope.ports else None + + def read_connection_peers(self, connections: list[ProvisioningConnection]) -> dict[int, ScopeData | None]: + cache: dict[tuple[str, str | None], ScopeData | None] = {} + peers: dict[int, ScopeData | None] = {} + for index, connection in enumerate(connections): + target = connection.peer.target + key = (target.device_id, target.module_id if isinstance(target, ModuleTarget) else None) + if key not in cache: + cache[key] = self.read_peer_scope(target) + peers[index] = cache[key] + if peers[index] is not None: + resolve_port(peers[index], [connection.peer.port], key=f"connections.{index}.peer") + return peers + + def check_peers(self, connections: list[ProvisioningConnection], peers: dict[int, ScopeData | None]) -> None: + checked: set[tuple[str, str | None]] = set() + for index, baseline in peers.items(): + if baseline is None: + continue # A newly available peer is deliberately picked up only by a new plan. + key = (baseline.device_id, baseline.module_id) + if key in checked: + continue + current = self.read_peer_scope(connections[index].peer.target) + if current is None or current.fingerprint != baseline.fingerprint: + raise ProvisioningConflictError( + f"Peer topology '{baseline.module_id or baseline.device_id}' changed; replan." + ) + checked.add(key) + + def read_edges(self, edge_ids: list[str]) -> dict[str, InspectApiEdgeForm]: + if not edge_ids: + return {} + return {key: item.edge for key, item in self._api.lookup_edges(edge_ids).data.items()} + + def plan_connections( + self, + work: TopologyWork, + scope: ScopeData, + connections: list[ProvisioningConnection], + peers: dict[int, ScopeData | None], + module_position: str | None, + ) -> TopologyWork: + resolved_inputs = [ + connection.model_copy( + update={"local": _port_selector(connection.local, module_position, f"connections.{index}.local")} + ) + if isinstance(connection.local, PortSelector) + else connection + for index, connection in enumerate(connections) + ] + legacy = [ + PortBinding( + **binding.model_dump(), + device_id=scope.device_id, + module_id=next(port.module_id for port in scope.ports if port.id == binding.port_id), + ) + for binding in work.bindings + ] + pending = resolve_connections(resolved_inputs, scope, peers, [*legacy, *work.port_bindings]) + compared = compare_edges(pending, self.read_edges(list(pending.edges))) + return work.model_copy( + update={ + "connections": compared, + "topology_operations": [*work.topology_operations, *compared.operations], + } + ) + + def check_edges(self, work: ConnectionWork) -> None: + current = self.read_edges(list(work.edges)) + for edge_id, edge in work.edges.items(): + if current.get(edge_id) != edge.baseline: + raise ProvisioningConflictError(f"Edge '{edge_id}' changed since planning; create a new plan.") + + def verify_edges(self, work: ConnectionWork) -> list[str]: + written = {key: edge for key, edge in work.edges.items() if edge.operation is not None} + current = self.read_edges(list(written)) + return [ + f"{edge_id}.{path}" + for edge_id, edge in written.items() + for path, value in edge.expected.items() + if edge_id not in current or not same_edge_value(path, edge_value(current[edge_id], path), value) + ] + # --- Writes --- def add_to_topology(self, device_id: str) -> None: + self.check_deadline() if not self._app.add_devices_to_topology([device_id], sync=False): raise TopologyNotReadyError(f"addDevices reported failure for '{device_id}' (not discovered yet?).") def sync(self, device_id: str, *, add_only: bool) -> None: + self.check_deadline() if not self._app.sync_devices([device_id], add_only=add_only): - raise BlueprintError(f"syncDevices failed for '{device_id}'; see the server log / Inspect sync dialog.") + raise ProvisioningError(f"syncDevices failed for '{device_id}'; see the server log / Inspect sync dialog.") def commit(self, work: TopologyWork) -> Any: with self._app.transaction() as tx: @@ -256,6 +400,14 @@ def commit(self, work: TopologyWork) -> Any: tx.update_device(work.device_id, intents=dict(work.device_intents)) for vertex_id, intents in sorted(work.vertex_intents.items()): tx.update_vertex(vertex_id, intents=dict(intents)) + for edge_id, edge in work.connections.edges.items(): + if edge.operation is None: + continue + if edge.baseline is None: + tx.connect(edge.from_vertex, edge.to_vertex, bidirectional=False, **edge.intents) + else: + tx.update_edge(edge_id, intents=dict(edge.intents)) + self.check_edges(work.connections) return tx.commit() def assign_tag(self, tag_id: str, module_id: str) -> None: @@ -280,10 +432,14 @@ def _detail(self, device_id: str) -> InspectApiNodeStatusItem | None: def _device_form(self, device_id: str, *, required: bool) -> Any | None: try: return self._api.lookup_inspect_device(device_id).data.fields - except Exception as exc: + except InspectEntityNotFoundError as exc: if not required: return None - raise BlueprintError( + raise TopologyNotReadyError(f"Inspect edit form of device '{device_id}' is not available yet.") from exc + except ProvisioningError: + raise + except Exception as exc: + raise ProvisioningError( f"Could not read the Inspect edit form of device '{device_id}': {type(exc).__name__}: {exc}" ) from exc @@ -312,7 +468,7 @@ def resolve_interfaces( if port.factory_label == label and any(by_id[v].kind == "ip" for v in port.vertex_ids if v in by_id) ] if len(ports) > 1: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Interface '{key}': factory label '{label}' matches {len(ports)} ports " f"({', '.join(p.id for p in ports)}) in {_scope_label(scope)}." ) @@ -331,7 +487,7 @@ def resolve_interfaces( break else: # No ports at all means discovery has not produced them yet; a miss among ports is final. - error = TopologyNotReadyError if not scope.ports else BlueprintTargetError + error = TopologyNotReadyError if not scope.ports else ProvisioningTargetError raise error( f"Interface '{key}': no IP port in {_scope_label(scope)} matches {', '.join(map(repr, attempted))}." ) @@ -344,6 +500,7 @@ def build_context( bindings: list[InterfaceBinding], owner: DriverContext | None, inventory: DriverContext | None, + port_bindings: list[PortBinding] | None = None, ) -> ProcessingContext: return ProcessingContext( source=source, @@ -355,6 +512,7 @@ def build_context( ports=scope.ports, vertices=scope.vertices, interfaces=tuple(bindings), + port_bindings=tuple(port_bindings or []), owner=owner, inventory=inventory if inventory is None or owner is None or inventory.inventory_id != owner.inventory_id @@ -376,30 +534,36 @@ def compute_topology_work( """Run the processor on the detached scope and turn proposals into exact, compared edits.""" config = resolved.config bindings = resolve_interfaces(config.ip_vertex_mapping, scope, source.module_position) - context = build_context(scope, source, bindings, owner, inventory) + port_bindings = [ + resolve_port(scope, [_port_selector(selector, source.module_position, key) for selector in candidates], key=key) + for key, candidates in (config.port_mapping or {}).items() + ] + context = build_context(scope, source, bindings, owner, inventory, port_bindings) result = _run_processor(resolved, context) is_module = scope.module_id is not None processor_source = f"processor:{resolved.processor_id}" if is_module and result.device is not None and result.device.managed(): - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Processor '{resolved.processor_id}' proposed device-wide changes for a module target." ) if not is_module and result.module is not None and result.module.managed(): - raise BlueprintTargetError(f"Processor '{resolved.processor_id}' proposed module changes for a device target.") + raise ProvisioningTargetError( + f"Processor '{resolved.processor_id}' proposed module changes for a device target." + ) if is_module and config.device is not None and config.device.managed(): - raise BlueprintTargetError( + raise ProvisioningTargetError( "'topology.device' settings cannot be applied to a module target; use a device target." ) if not is_module and config.module is not None and config.module.managed(): - raise BlueprintTargetError("'topology.module' settings require a module target.") + raise ProvisioningTargetError("'topology.module' settings require a module target.") # Vertex proposals: processor < naming < explicit overrides. vertex_desired: dict[str, dict[str, tuple[Any, str]]] = defaultdict(dict) endpoints: dict[str, EndpointIdentity] = {} for edit in result.vertices: if scope.vertex(edit.vertex_id) is None: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Processor '{resolved.processor_id}' proposed vertex '{edit.vertex_id}' outside {_scope_label(scope)}." ) for field, value in edit.fields.managed().items(): @@ -419,7 +583,7 @@ def compute_topology_work( for vertex_id, endpoint in endpoints.items(): record = scope.vertex(vertex_id) assert record is not None - facts = name_context(source, endpoint=endpoint, vertex=record) + facts = name_context(source, endpoint=endpoint, vertex=record, inputs=resolved.inputs) for entry, field in (("endpoint_label", "label"), ("endpoint_description", "description")): expression = getattr(naming, entry) if expression is not None: @@ -435,10 +599,12 @@ def compute_topology_work( record = scope.vertex(vertex_id) assert record is not None if vertex_id in scope.without_form: - raise BlueprintTargetError(f"Vertex '{vertex_id}' has no edit form on the server; it cannot be configured.") + raise ProvisioningTargetError( + f"Vertex '{vertex_id}' has no edit form on the server; it cannot be configured." + ) for field, kind in _KIND_SPECIFIC_FIELDS.items(): if field in fields and record.kind != kind: - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( f"'{field}' is only supported on {kind} vertices ('{vertex_id}' is '{record.kind}')." ) @@ -455,7 +621,7 @@ def compute_topology_work( for k, v in config.device.managed().items() } ) - facts = name_context(source) + facts = name_context(source, inputs=resolved.inputs) for entry, field in (("device_label", "label"), ("device_description", "description")): expression = getattr(naming, entry) if expression is not None: @@ -469,7 +635,12 @@ def compute_topology_work( work = _compare(scope, vertex_desired, endpoints, device_desired, module_tags) _check_label_collisions(scope, work, allow=allow_label_collisions) return work.model_copy( - update={"bindings": bindings, "diagnostics": list(result.diagnostics), "fingerprint": scope.fingerprint} + update={ + "bindings": bindings, + "port_bindings": port_bindings, + "diagnostics": list(result.diagnostics), + "fingerprint": scope.fingerprint, + } ) @@ -492,7 +663,11 @@ def verify_topology(scope: ScopeData, work: TopologyWork) -> list[str]: def name_context( - source: SourceFacts, *, endpoint: EndpointIdentity | None = None, vertex: VertexRecord | None = None + source: SourceFacts, + *, + endpoint: EndpointIdentity | None = None, + vertex: VertexRecord | None = None, + inputs: dict[str, JsonValue] | None = None, ) -> NameContext: """Naming facts for a source entity (and optionally one endpoint vertex).""" return NameContext( @@ -501,6 +676,7 @@ def name_context( endpoint=endpoint.model_dump() if endpoint is not None else {}, vertex={"factory_label": vertex.factory_label, "id": vertex.id} if vertex is not None else {}, attributes=source.attributes, + inputs=inputs or {}, ) @@ -542,7 +718,7 @@ def _run_processor(resolved: ResolvedTopology, context: ProcessingContext) -> Pr return ProcessorResult() try: raw = resolved.processor_cls().process(context, resolved.params) - except BlueprintError: + except ProvisioningError: raise except Exception as exc: raise ProcessorInputError(f"Processor '{resolved.processor_id}' failed: {type(exc).__name__}: {exc}") from exc @@ -612,7 +788,7 @@ def _compare( baseline = module.local_tags if module is not None else None if baseline is None: if not isinstance(spec, TagDelta) or spec.remove: - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( f"Local tags of module '{scope.module_id}' cannot be distinguished from inherited tags on this " "server; only additive tag changes ({add: [...]}) are supported for it." ) @@ -666,7 +842,7 @@ def _check_label_collisions(scope: ScopeData, work: TopologyWork, *, allow: bool collisions = {label: ids for label, ids in labels.items() if len(ids) > 1 and changed.intersection(ids)} if collisions: detail = "; ".join(f"'{label}' on {', '.join(sorted(ids))}" for label, ids in sorted(collisions.items())) - raise BlueprintValidationError( + raise ProvisioningValidationError( ValidationIssue( path="naming.endpoint_label", message=f"Endpoint label collision in device '{scope.device_id}': {detail}. Adjust the naming rule " @@ -679,7 +855,7 @@ def _check_label_collisions(scope: ScopeData, work: TopologyWork, *, allow: bool def _select_override(override: Any, scope: ScopeData, module_position: str | None, index: int) -> str: if override.vertex_id is not None: if scope.vertex(override.vertex_id) is None: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Vertex override #{index}: '{override.vertex_id}' is not in {_scope_label(scope)}." ) return override.vertex_id @@ -693,7 +869,7 @@ def _select_override(override: Any, scope: ScopeData, module_position: str | Non ] if len(matches) != 1: found = ", ".join(v.id for v in matches) or "none" - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Vertex override #{index}: factory label '{label}' must match exactly one vertex in " f"{_scope_label(scope)}; matched {found}. Add 'kind'/'direction' or use 'vertex_id'." ) @@ -703,7 +879,7 @@ def _select_override(override: Any, scope: ScopeData, module_position: str | Non def _substitute_position(template: str, module_position: str | None, owner: str) -> str: def replace(match: re.Match[str]) -> str: if match.group(1) != "module.position": - raise BlueprintValidationError( + raise ProvisioningValidationError( ValidationIssue( path=owner, message=f"Unsupported placeholder '{{{match.group(1)}}}' (only '{{module.position}}').", @@ -711,10 +887,10 @@ def replace(match: re.Match[str]) -> str: ) ) if module_position is None: - raise BlueprintValidationError( + raise ProvisioningValidationError( ValidationIssue( path=owner, - message="'{module.position}' requires BlueprintDevice.module_position.", + message="'{module.position}' requires ProvisioningDevice.module_position.", code="mapping.missing", ) ) @@ -724,6 +900,14 @@ def replace(match: re.Match[str]) -> str: return _POSITION_TOKEN.sub(replace, escaped).replace("\x00", "{").replace("\x01", "}") +def _port_selector(selector: PortSelector, module_position: str | None, owner: str) -> PortSelector: + if selector.factory_label is None: + return selector + return selector.model_copy( + update={"factory_label": _substitute_position(selector.factory_label, module_position, owner)} + ) + + def _resolve_tags(spec: Any, current: Any, owner: str) -> list[str]: current_list = list(current or []) if isinstance(spec, TagDelta): @@ -731,7 +915,7 @@ def _resolve_tags(spec: Any, current: Any, owner: str) -> list[str]: return kept + [tag for tag in dict.fromkeys(spec.add) if tag not in kept] if isinstance(spec, (list, tuple)): return list(dict.fromkeys(spec)) - raise BlueprintValidationError(f"Invalid tag specification for {owner}.") + raise ProvisioningValidationError(f"Invalid tag specification for {owner}.") def _same(field: str, current: Any, desired: Any) -> bool: @@ -875,7 +1059,7 @@ def _check_action(response: Any, action: str, tag_id: str) -> None: if response.header.ok and response.data.ok: return detail = "; ".join(m for m in list(response.data.msg) + list(response.header.msg) if m) or "rejected" - raise BlueprintError(f"{action} failed for tag '{tag_id}': {detail}") + raise ProvisioningError(f"{action} failed for tag '{tag_id}': {detail}") __all__ = [ diff --git a/src/videoipath_automation_tool/blueprints/inventory.py b/src/videoipath_automation_tool/provisioning/inventory.py similarity index 90% rename from src/videoipath_automation_tool/blueprints/inventory.py rename to src/videoipath_automation_tool/provisioning/inventory.py index 92f6163..a16afcb 100644 --- a/src/videoipath_automation_tool/blueprints/inventory.py +++ b/src/videoipath_automation_tool/provisioning/inventory.py @@ -24,24 +24,26 @@ from pydantic import BaseModel, ConfigDict, SecretStr +from videoipath_automation_tool.apps.inventory.errors import InventoryStatusUnavailableError +from videoipath_automation_tool.apps.inventory.model.device_status import DeviceStatus from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice from videoipath_automation_tool.apps.inventory.model.inventory_device_configuration import Auth -from videoipath_automation_tool.blueprints.errors import ( - BlueprintCapabilityError, - BlueprintConflictError, - BlueprintError, - BlueprintTargetError, - BlueprintValidationError, +from videoipath_automation_tool.provisioning.errors import ( + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningError, + ProvisioningTargetError, + ProvisioningValidationError, ValidationIssue, ) -from videoipath_automation_tool.blueprints.models import ( +from videoipath_automation_tool.provisioning.models import ( AlternativeAddress, - BlueprintDevice, CatalogId, FieldChange, PlannedOperation, + ProvisioningDevice, ) -from videoipath_automation_tool.blueprints.resolution import ResolvedInventory +from videoipath_automation_tool.provisioning.resolution import ResolvedInventory REDACTED = "********" _SENSITIVE_TOKENS = frozenset({"password", "passwd", "secret", "token", "community", "key"}) @@ -101,7 +103,14 @@ def read(self, inventory_id: str) -> InventoryDevice: try: return self._app.get_device(device_id=inventory_id, config_only=True) except ValueError as exc: - raise BlueprintTargetError(f"Inventory record '{inventory_id}' was not found: {exc}") from None + raise ProvisioningTargetError(f"Inventory record '{inventory_id}' was not found: {exc}") from None + + def read_status(self, device: InventoryDevice) -> DeviceStatus | None: + """One fresh status request, without the Inventory get-device retry loop.""" + try: + return self._app.refresh_device_status(device=device).status + except InventoryStatusUnavailableError: + return None def ids_with_label(self, label: str) -> list[str]: found = self._app.find_device_id_by_label(label, label_search_mode="user_defined_label_only") @@ -110,14 +119,14 @@ def ids_with_label(self, label: str) -> list[str]: def ids_by_addresses(self, addresses: list[str]) -> dict[str, list[str]]: """Device ids per address (normalized comparison), in one read. - A response with no item list raises :class:`BlueprintError`. + A response with no item list raises :class:`ProvisioningError`. """ if not addresses: return {} try: return self._app.find_device_ids_by_addresses(addresses) except ValueError as exc: - raise BlueprintError(f"Could not read Inventory addresses for the conflict check: {exc}") from exc + raise ProvisioningError(f"Could not read Inventory addresses for the conflict check: {exc}") from exc def resolve_snmp(self, reference: str | CatalogId) -> str: """Exact SNMP configuration id for a label or ``{id: ...}``; missing/ambiguous fail.""" @@ -126,14 +135,14 @@ def resolve_snmp(self, reference: str | CatalogId) -> str: return self._snmp_cache[key] if isinstance(reference, CatalogId): if reference.id != "default" and self._app.get_global_snmp_config_label_by_id(reference.id) is None: - raise BlueprintTargetError(f"SNMP configuration id '{reference.id}' does not exist.") + raise ProvisioningTargetError(f"SNMP configuration id '{reference.id}' does not exist.") resolved = reference.id else: found = _as_list(self._app.get_global_snmp_config_id_by_label(reference)) if not found: - raise BlueprintTargetError(f"No SNMP configuration labelled '{reference}'.") + raise ProvisioningTargetError(f"No SNMP configuration labelled '{reference}'.") if len(found) > 1: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"SNMP configuration label '{reference}' is ambiguous ({', '.join(found)}); use {{id: ...}}." ) resolved = found[0] @@ -152,7 +161,7 @@ def update(self, candidate: InventoryDevice) -> InventoryDevice: def plan_inventory( gateway: InventoryGateway, resolved: ResolvedInventory, - device: BlueprintDevice, + device: ProvisioningDevice, *, label: str | None, description: str | None, @@ -169,18 +178,18 @@ def plan_inventory( if device.inventory_id is not None: current = gateway.read(device.inventory_id) if current.driver_id != resolved.driver_id: - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( f"Inventory record '{device.inventory_id}' uses driver '{current.driver_id}', the blueprint selects " f"'{resolved.driver_id}'. Driver migration is not automatic; migrate explicitly with the Inventory API." ) else: if device.management_address is None: - raise BlueprintTargetError( + raise ProvisioningTargetError( f"Creating an Inventory record for '{device.key}' requires 'management_address' " "(or bind an existing record with 'inventory_id')." ) if "label" not in desired: - raise BlueprintValidationError( + raise ProvisioningValidationError( ValidationIssue( path="naming.inventory_label", message="Creating an Inventory record requires an Inventory label; the naming entry is disabled.", @@ -208,7 +217,7 @@ def plan_inventory( return work, current -def recheck_conflicts(gateway: InventoryGateway, device: BlueprintDevice, work: InventoryWork) -> None: +def recheck_conflicts(gateway: InventoryGateway, device: ProvisioningDevice, work: InventoryWork) -> None: """Repeat the label/address conflict check immediately before a write.""" _check_conflicts(gateway, device, work.desired, work.changes, own_id=work.inventory_id) @@ -229,7 +238,7 @@ def check_baseline(work: InventoryWork, fresh: InventoryDevice) -> None: current = _baseline(fresh, work.desired) changed = sorted(key for key in work.baseline if current.get(key) != work.baseline[key]) if changed: - raise BlueprintConflictError( + raise ProvisioningConflictError( f"Inventory record '{work.inventory_id}' changed since planning ({', '.join(changed)}); replan." ) @@ -270,7 +279,7 @@ def is_sensitive(key: str) -> bool: def _desired_values( gateway: InventoryGateway, resolved: ResolvedInventory, - device: BlueprintDevice, + device: ProvisioningDevice, *, label: str | None, description: str | None, @@ -332,7 +341,7 @@ def _alt_with_auth(entry: AlternativeAddress) -> dict[str, Any]: def _check_conflicts( gateway: InventoryGateway, - device: BlueprintDevice, + device: ProvisioningDevice, desired: dict[str, DesiredValue], changes: list[FieldChange], *, @@ -357,7 +366,7 @@ def _check_conflicts( conflicts.append(f"address '{address}' is used by {', '.join(others)}") if conflicts: hint = "" if own_id else " Bind the existing record explicitly with 'inventory_id' if it is the same device." - raise BlueprintTargetError(f"Inventory conflict for '{device.key}': {'; '.join(conflicts)}.{hint}") + raise ProvisioningTargetError(f"Inventory conflict for '{device.key}': {'; '.join(conflicts)}.{hint}") def _baseline(device: InventoryDevice, desired: dict[str, DesiredValue]) -> dict[str, str]: diff --git a/src/videoipath_automation_tool/blueprints/loader.py b/src/videoipath_automation_tool/provisioning/loader.py similarity index 97% rename from src/videoipath_automation_tool/blueprints/loader.py rename to src/videoipath_automation_tool/provisioning/loader.py index b19e2fb..0d93606 100644 --- a/src/videoipath_automation_tool/blueprints/loader.py +++ b/src/videoipath_automation_tool/provisioning/loader.py @@ -19,7 +19,7 @@ import yaml -from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue MAX_DOCUMENT_BYTES = 1024 * 1024 MAX_DEPTH = 32 @@ -171,8 +171,8 @@ def _scalar(self, node: Any) -> Any: raise _error(f"unsupported scalar tag or value for tag {tag!r}", self._source, node.start_mark) -def _error(message: str, source: str | None, mark: Any = None) -> BlueprintValidationError: - return BlueprintValidationError( +def _error(message: str, source: str | None, mark: Any = None) -> ProvisioningValidationError: + return ProvisioningValidationError( ValidationIssue( message=message, code="yaml.invalid", diff --git a/src/videoipath_automation_tool/blueprints/models.py b/src/videoipath_automation_tool/provisioning/models.py similarity index 73% rename from src/videoipath_automation_tool/blueprints/models.py rename to src/videoipath_automation_tool/provisioning/models.py index b48e40d..94098d9 100644 --- a/src/videoipath_automation_tool/blueprints/models.py +++ b/src/videoipath_automation_tool/provisioning/models.py @@ -10,6 +10,7 @@ from __future__ import annotations +from collections.abc import Mapping, Sequence from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Literal @@ -24,28 +25,33 @@ SecretStr, StringConstraints, ValidationError, + field_validator, model_validator, ) from videoipath_automation_tool.apps.inspect.model.common import ( + InspectConfigPriority, InspectIconSize, InspectIconType, + InspectRedundancyMode, InspectSdpStrategy, InspectSipsMode, ) -from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue -from videoipath_automation_tool.blueprints.loader import parse_yaml, read_blueprint_file -from videoipath_automation_tool.blueprints.naming import ( +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue +from videoipath_automation_tool.provisioning.inputs import InputDefinition, InputName +from videoipath_automation_tool.provisioning.loader import parse_yaml, read_blueprint_file +from videoipath_automation_tool.provisioning.naming import ( INVENTORY_NAMING_ENTRIES, TOPOLOGY_NAMING_ENTRIES, BlueprintNaming, ) +from videoipath_automation_tool.provisioning.templates import template_models, validate_document_references from videoipath_automation_tool.utils.cross_app_utils import normalize_address from videoipath_automation_tool.validators.device_id import validate_device_id if TYPE_CHECKING: from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice - from videoipath_automation_tool.blueprints.processors import ProcessorRegistry + from videoipath_automation_tool.provisioning.processors import ProcessorRegistry NonEmptyStr = Annotated[str, StringConstraints(min_length=1)] VariantName = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")] @@ -162,6 +168,38 @@ def _one_selector(self) -> VertexOverride: return self +class PortSelector(_StrictModel): + """An exact port id or factory label, optionally restricted to a vertex kind.""" + + port_id: NonEmptyStr | None = None + factory_label: NonEmptyStr | None = None + kind: NonEmptyStr | None = None + + @model_validator(mode="after") + def _one_selector(self) -> PortSelector: + if (self.port_id is None) == (self.factory_label is None): + raise ValueError("specify exactly one of 'port_id' or 'factory_label'") + return self + + +class EdgePatch(_PatchModel): + """Managed fields of a concrete connection, applied to each requested direction.""" + + label: str | None = None + description: str | None = None + weight: int | None = None + capacity: int | None = None + bandwidth: float | None = None + redundancy_mode: InspectRedundancyMode | None = None + conflict_priority: InspectConfigPriority | int | None = None + include_formats: list[NonEmptyStr] | None = None + exclude_formats: list[NonEmptyStr] | None = None + bandwidth_weight_factor: int | None = None + weight_per_service: int | None = None + active: bool | None = None + tags: TagSpec | None = None + + class VertexProcessorSpec(_StrictModel): """Processor declaration. ``processor_type`` may be inherited from ``default`` by a variant; it is required on the resolved configuration.""" @@ -196,7 +234,7 @@ class GenericSettings(_PatchModel): class InventorySettings(_StrictModel): - """Inventory settings that may also be supplied per instance (``BlueprintDevice.inventory_overrides``).""" + """Inventory settings that may also be supplied per instance (``ProvisioningDevice.inventory_overrides``).""" active: bool | None = None generic_settings: GenericSettings | None = None @@ -225,6 +263,7 @@ class TopologyConfig(_StrictModel): device: DevicePatch | None = None module: ModulePatch | None = None ip_vertex_mapping: dict[NonEmptyStr, Annotated[list[NonEmptyStr], Field(min_length=1)]] | None = None + port_mapping: dict[NonEmptyStr, Annotated[list[PortSelector], Field(min_length=1)]] | None = None vertex_processor: VertexProcessorSpec | None = None naming: BlueprintNaming | None = None vertices: list[VertexOverride] | None = None @@ -232,30 +271,55 @@ class TopologyConfig(_StrictModel): @model_validator(mode="after") def _topology_naming_only(self) -> TopologyConfig: _check_naming_entries(self.naming, TOPOLOGY_NAMING_ENTRIES, "topology") + if set(self.ip_vertex_mapping or {}) & set(self.port_mapping or {}): + raise ValueError("ip_vertex_mapping and port_mapping must have distinct keys") return self +InventoryTemplate, TopologyTemplate = template_models(InventoryConfig, TopologyConfig) + + +class BlueprintConfiguration(_StrictModel): + """A configuration or overlay. Section keys are structural, not input references.""" + + inventory: InventoryTemplate | None = None + topology: TopologyTemplate | None = None + + @model_validator(mode="before") + @classmethod + def _reject_null_sections(cls, data: Any) -> Any: + if isinstance(data, dict) and any(value is None for value in data.values()): + raise ValueError("omit a section instead of setting it to null") + return data + + class Blueprint(_StrictModel): - """A reusable, type-level configuration with ``default`` entries and optional named variants. + """A reusable configuration with typed inputs and ordered variant overlays. Load with :meth:`load` (file), :meth:`from_yaml` (text/bytes), or :meth:`from_dict` (mapping). - Loading never connects to a server. A named variant is a patch over its section's ``default``. + Loading never connects to a server. Variants patch ``defaults`` in caller-selected order. """ schema_version: Literal[1] - inventory: dict[VariantName, InventoryConfig] | None = None - topology: dict[VariantName, TopologyConfig] | None = None + inputs: dict[InputName, InputDefinition] = Field(default_factory=dict) + defaults: BlueprintConfiguration + variants: dict[VariantName, BlueprintConfiguration] = Field(default_factory=dict) _source: str | None = PrivateAttr(default=None) _locations: dict[tuple[Any, ...], tuple[int, int]] = PrivateAttr(default_factory=dict) + @field_validator("schema_version", mode="before") + @classmethod + def _version_type(cls, value: Any) -> Any: + if type(value) is not int: + raise ValueError("schema_version must be the integer 1") + return value + @model_validator(mode="after") def _sections(self) -> Blueprint: - if self.inventory is None and self.topology is None: - raise ValueError("a blueprint needs an 'inventory' or a 'topology' section") - for name, section in (("inventory", self.inventory), ("topology", self.topology)): - if section is not None and "default" not in section: - raise ValueError(f"the '{name}' section must contain a 'default' entry") + if self.defaults.inventory is None and self.defaults.topology is None: + raise ValueError("defaults needs an 'inventory' or a 'topology' section") + validate_document_references(self.model_dump(by_alias=True, exclude_unset=True), self.inputs) return self # --- Construction --- @@ -276,22 +340,30 @@ def from_yaml(cls, text: str | bytes, *, source: str | None = None) -> Blueprint def from_dict(cls, data: dict[str, Any], *, source: str | None = None) -> Blueprint: """Build from a mapping (e.g. produced by caller preprocessing). The mapping is not retained.""" if not isinstance(data, dict): - raise BlueprintValidationError(ValidationIssue(message="a blueprint must be a mapping", source=source)) + raise ProvisioningValidationError(ValidationIssue(message="a blueprint must be a mapping", source=source)) return cls._from_data(data, source=source, locations={}) # --- Validation and schema --- - def validate_full(self, registry: ProcessorRegistry | None = None) -> None: - """Resolve and validate *every* variant (including unused ones) against the selected driver - models, naming rules, and — when ``registry`` is given — processor parameter schemas. + def validate_full( + self, + registry: ProcessorRegistry | None = None, + *, + inputs: Mapping[str, JsonValue] | None = None, + variants: Sequence[str] | None = None, + ) -> None: + """Validate defaults and each individual variant, or one explicit ordered combination. + + ``inputs`` supplies declared values. With ``variants``, only that combination is checked. + Validation checks driver models, naming, and registered processor parameter schemas. Processors absent from ``registry`` are reported as ``processor.unavailable`` issues. Raises: - BlueprintValidationError: listing all issues found. + ProvisioningValidationError: listing all issues found. """ - from videoipath_automation_tool.blueprints.resolution import validate_all_variants + from videoipath_automation_tool.provisioning.resolution import validate_all_variants - validate_all_variants(self, registry) + validate_all_variants(self, registry, inputs=inputs, variants=variants) @classmethod def json_schema( @@ -299,13 +371,12 @@ def json_schema( ) -> dict[str, Any]: """JSON Schema of the document; with ``registry``, adds parameter schemas per processor id. ``restrict_processor_types`` additionally limits ``processor_type`` to the registered ids.""" - from videoipath_automation_tool.blueprints.resolution import blueprint_json_schema + from videoipath_automation_tool.provisioning.resolution import blueprint_json_schema return blueprint_json_schema(registry, restrict_processor_types=restrict_processor_types) - def variant_names(self, section: Literal["inventory", "topology"]) -> list[str]: - entries = getattr(self, section) - return list(entries) if entries else [] + def variant_names(self) -> list[str]: + return list(self.variants) def location_of(self, path: tuple[Any, ...]) -> tuple[str | None, int | None, int | None]: """``(source, line, column)`` of a document path when loaded from YAML.""" @@ -325,9 +396,24 @@ def _from_data( try: blueprint = cls.model_validate(data) except ValidationError as exc: - raise BlueprintValidationError( + raise ProvisioningValidationError( issues_from_pydantic(exc, data=data, source=source, locations=locations) ) from None + except ProvisioningValidationError as exc: + issues = [] + for issue in exc.issues: + keys = tuple(int(part) if part.isdigit() else part for part in issue.path.split(".")) + position = _nearest_location(keys, locations) + issues.append( + issue.model_copy( + update={ + "source": source, + "line": position[0] if position else None, + "column": position[1] if position else None, + } + ) + ) + raise ProvisioningValidationError(issues) from None blueprint._source = source blueprint._locations = dict(locations) return blueprint @@ -364,7 +450,23 @@ class ModuleTarget(_StrictModel): TopologyTarget = DeviceTarget | ModuleTarget -class BlueprintDevice(_StrictModel): +class PeerEndpoint(_StrictModel): + """An existing VideoIPath device/module and one of its ports.""" + + target: TopologyTarget + port: PortSelector + + +class ProvisioningConnection(_StrictModel): + """Concrete external connection; directions are relative to the local device.""" + + local: NonEmptyStr | PortSelector + peer: PeerEndpoint + direction: Literal["auto", "outgoing", "incoming", "bidirectional"] = "auto" + fields: EdgePatch = Field(default_factory=EdgePatch) + + +class ProvisioningDevice(_StrictModel): """Instance facts and explicit VideoIPath bindings for one source entity. ``key`` is an opaque caller correlation key (never written to VideoIPath). ``inventory_id`` binds @@ -374,6 +476,8 @@ class BlueprintDevice(_StrictModel): set and also replaces per-address credentials: a plain string carries none, so an existing credential for that address is cleared unless it is repeated as an ``AlternativeAddress``. ``attributes`` are caller-owned facts available to naming (scalar leaves) and processors. + ``connections`` declares concrete external connections and their managed edge fields. + Missing peer discovery is reported as deferred; it never creates the peer device. """ key: NonEmptyStr @@ -387,9 +491,10 @@ class BlueprintDevice(_StrictModel): module_position: NonEmptyStr | None = None inventory_overrides: InventorySettings | None = None attributes: dict[str, JsonValue] = Field(default_factory=dict) + connections: list[ProvisioningConnection] = Field(default_factory=list) @model_validator(mode="after") - def _unique_addresses(self) -> BlueprintDevice: + def _unique_addresses(self) -> ProvisioningDevice: seen: dict[str, Any] = {} if self.management_address is not None: seen[normalize_address(self.management_address)] = None @@ -402,7 +507,7 @@ def _unique_addresses(self) -> BlueprintDevice: return self @classmethod - def from_inventory(cls, device: InventoryDevice, *, key: str | None = None) -> BlueprintDevice: + def from_inventory(cls, device: InventoryDevice, *, key: str | None = None) -> ProvisioningDevice: """Convenience binding for an existing Inventory device (id, label, description only).""" configuration = device.configuration return cls( @@ -457,22 +562,28 @@ class ApplyOptions(_StrictModel): ``sync``: ``"none"`` requires existing topology; ``"add_only"`` (default) adds/syncs only new elements; ``"reconcile"`` permits a full sync. Service conflicts are always handled strictly. - ``discovery_timeout`` / ``poll_interval`` bound the wait for driver discovery (seconds). + ``inventory_ready_timeout`` bounds the wait for reachability before deferred topology work. + ``topology_ready_timeout`` is a separate budget for topology addition, sync, and required data. + Both use ``poll_interval`` (seconds). ``require_reachable=False`` bypasses the Inventory gate. ``naming_collisions="allow"`` permits duplicate endpoint labels within a device. ``write_credentials=True`` writes the supplied credentials to an existing Inventory record even when nothing else changes (secrets are masked on read, so a plan cannot tell whether they differ). """ sync: SyncPolicy = "add_only" - discovery_timeout: Annotated[float, Field(gt=0)] = 30.0 - poll_interval: Annotated[float, Field(gt=0)] = 1.0 + inventory_ready_timeout: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 10.0 + topology_ready_timeout: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 60.0 + require_reachable: bool = True + poll_interval: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 1.0 naming_collisions: Literal["reject", "allow"] = "reject" write_credentials: bool = False # --- Plan and result records --- -PhaseName = Literal["inventory", "discovery", "topology_sync", "topology", "module_tags", "verification"] +PhaseName = Literal[ + "inventory", "inventory_readiness", "topology_sync", "discovery", "topology", "module_tags", "verification" +] class FieldChange(BaseModel): @@ -491,7 +602,7 @@ class PlannedOperation(BaseModel): model_config = ConfigDict(frozen=True) action: Literal["create", "update", "assign_tag", "unassign_tag", "add_to_topology", "sync"] - entity_kind: Literal["inventory", "device", "vertex", "module"] + entity_kind: Literal["inventory", "device", "vertex", "module", "edge"] entity_id: str | None = None changes: list[FieldChange] = Field(default_factory=list) endpoint: EndpointIdentity | None = None @@ -518,6 +629,25 @@ class InterfaceBinding(BaseModel): out_vertex_id: str | None = None +class PortBinding(InterfaceBinding): + """A generic port binding including its owning device/module and actual vertex ids.""" + + device_id: str + module_id: str | None = None + + +class ConnectionState(_StrictModel): + """One input connection's resolution and execution state, including unresolved work.""" + + index: int + connection: ProvisioningConnection + local: PortBinding | None = None + peer: PortBinding | None = None + edge_ids: list[str] = Field(default_factory=list) + status: Literal["planned", "no_change", "deferred", "completed", "not_run", "failed", "unknown"] + reason: str | None = None + + class PhaseResult(BaseModel): name: PhaseName status: Literal["completed", "no_change", "skipped", "failed", "unknown", "not_run", "planned", "deferred"] @@ -532,21 +662,24 @@ class ApplyResult(BaseModel): In a dry run (``dry_run=True``) nothing is written: ``status`` is ``planned`` when writes would run and ``no_change`` otherwise, and write phases report ``planned`` / ``deferred``. - ``replan_required`` means a topology-affecting Inventory update finished and apply stopped. + ``replan_required`` means a topology-affecting Inventory update finished and apply stopped, + or some connections still have undiscovered peers. ``status`` is ``partial`` (``planned`` on a dry run) and the call does not raise. Plan again - after the driver has rediscovered the device. + after the driver has rediscovered the device or the peer topology is available. """ status: Literal["succeeded", "no_change", "failed", "partial", "unknown", "planned"] = "no_change" dry_run: bool = False replan_required: bool = False - """True when apply stopped after a topology-affecting Inventory update and a new plan is required.""" + """A new plan is needed after Inventory rediscovery or to resolve open peer connections.""" source_key: str inventory_id: str | None = None topology_device_id: str | None = None module_id: str | None = None phases: list[PhaseResult] = Field(default_factory=list) interface_bindings: list[InterfaceBinding] = Field(default_factory=list) + port_bindings: list[PortBinding] = Field(default_factory=list) + connections: list[ConnectionState] = Field(default_factory=list) materialized: bool = False diagnostics: list[Diagnostic] = Field(default_factory=list) verification: Literal["confirmed", "unconfirmed", "not_applicable"] = "not_applicable" @@ -656,13 +789,15 @@ def _friendly_message(error: Any) -> str: "ApplyOptions", "ApplyResult", "Blueprint", - "BlueprintDevice", + "BlueprintConfiguration", "CatalogId", + "ConnectionState", "Coordinates", "Credentials", "DevicePatch", "DeviceTarget", "Diagnostic", + "EdgePatch", "EndpointIdentity", "FieldChange", "GenericSettings", @@ -671,10 +806,15 @@ def _friendly_message(error: Any) -> str: "InventorySettings", "ModulePatch", "ModuleTarget", + "PeerEndpoint", "PhaseResult", "PlannedOperation", "PlannedPhase", + "PortBinding", + "PortSelector", "ProcessorResult", + "ProvisioningConnection", + "ProvisioningDevice", "Scope", "SnmpSelection", "TagDelta", diff --git a/src/videoipath_automation_tool/blueprints/naming.py b/src/videoipath_automation_tool/provisioning/naming.py similarity index 83% rename from src/videoipath_automation_tool/blueprints/naming.py rename to src/videoipath_automation_tool/provisioning/naming.py index d9d3671..4f7cc1b 100644 --- a/src/videoipath_automation_tool/blueprints/naming.py +++ b/src/videoipath_automation_tool/provisioning/naming.py @@ -14,12 +14,16 @@ from __future__ import annotations import re -from typing import Any, Protocol, runtime_checkable +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from pydantic import BaseModel, ConfigDict, InstanceOf, JsonValue, model_validator from pydantic import Field as PydanticField -from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ValidationIssue +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue + +if TYPE_CHECKING: + from videoipath_automation_tool.provisioning.inputs import InputDefinition NAMING_ENTRIES: tuple[str, ...] = ( "inventory_label", @@ -91,7 +95,7 @@ def __init__(self, parts: list[Any] | None = None, /, **data: Any) -> None: class NameContext(BaseModel): """The facts available to naming. Namespaces: ``device`` (key, label, description), ``module`` (position), ``endpoint`` (direction, media, index, engine, program, leg), ``vertex`` - (factory_label, id), and ``attributes`` (caller-owned scalar facts).""" + (factory_label, id), ``attributes`` (caller-owned scalar facts), and ``inputs`` (bound values).""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -100,6 +104,7 @@ class NameContext(BaseModel): endpoint: dict[str, Any] = PydanticField(default_factory=dict) vertex: dict[str, Any] = PydanticField(default_factory=dict) attributes: dict[str, JsonValue] = PydanticField(default_factory=dict) + inputs: dict[str, JsonValue] = PydanticField(default_factory=dict) def value(self, path: str) -> Any: """The value at ``path`` (e.g. ``"device.label"``), or ``None`` when absent.""" @@ -210,8 +215,55 @@ def validate_expression(entry: str, expr: Any) -> list[str]: return problems +def reference_paths(expr: Any) -> list[str]: + """Dotted paths used by a naming expression, for offline input-reference checks.""" + return [field.path for field in _fields_of(expr)] + + +def validate_naming_inputs( + naming: NamingScheme, + definitions: Mapping[str, InputDefinition], + values: Mapping[str, JsonValue], + *, + entries: frozenset[str] | None = None, +) -> None: + """Check naming inputs in the final layered scheme before reading server state.""" + issues: list[ValidationIssue] = [] + context = NameContext(inputs=dict(values)) + for entry, expression in naming.entries().items(): + if entries is not None and entry not in entries: + continue + for path in reference_paths(expression): + if not path.startswith("inputs."): + continue + parts = path.split(".")[1:] + definition = definitions.get(parts[0]) + message, code = None, "input.reference" + if definition is None: + message = "Naming reference must name a declared input." + elif parts[0] not in values: + message, code = "Required naming input is missing.", "input.missing" + else: + for part in parts[1:]: + if definition.type != "object": + message = "Naming input path traverses a non-object input." + break + properties = definition.properties or {} + if part not in properties: + if not definition.additional_properties: + message = "Unknown property in naming input reference." + break + definition = properties[part] + if isinstance(context.value(path), (dict, list)): + message = "Naming inputs must reference scalar leaves." + if message: + issues.append(ValidationIssue(path=f"naming.{entry}", message=message, code=code)) + if issues: + raise ProvisioningValidationError(issues) + + def render_name(entry: str, expr: Any, context: NameContext) -> str: - """Render ``expr`` for ``entry``; raises :class:`BlueprintValidationError` on missing required + """Render ``expr`` for ``entry``; raises :class:`ProvisioningValidationError` on missing required values, type/format errors, failed mappings, or an empty/control-character result.""" try: if isinstance(expr, (str, Text, Field, Join)): @@ -233,7 +285,7 @@ def render_name(entry: str, expr: Any, context: NameContext) -> str: # --- Internal --- -_NAMESPACES: frozenset[str] = frozenset({"device", "module", "endpoint", "vertex", "attributes"}) +_NAMESPACES: frozenset[str] = frozenset({"device", "module", "endpoint", "vertex", "attributes", "inputs"}) _KNOWN_PATHS: frozenset[str] = frozenset( { "device.key", @@ -250,7 +302,7 @@ def render_name(entry: str, expr: Any, context: NameContext) -> str: "vertex.id", } ) -_NON_ENDPOINT_NAMESPACES = frozenset({"device", "module", "attributes"}) +_NON_ENDPOINT_NAMESPACES = frozenset({"device", "module", "attributes", "inputs"}) _ENTRY_NAMESPACES: dict[str, frozenset[str]] = { "inventory_label": _NON_ENDPOINT_NAMESPACES, "inventory_description": _NON_ENDPOINT_NAMESPACES, @@ -277,8 +329,8 @@ class _RenderError(Exception): pass -def _naming_error(entry: str, message: str, code: str) -> BlueprintValidationError: - return BlueprintValidationError(ValidationIssue(path=f"naming.{entry}", message=message, code=code)) +def _naming_error(entry: str, message: str, code: str) -> ProvisioningValidationError: + return ProvisioningValidationError(ValidationIssue(path=f"naming.{entry}", message=message, code=code)) def _compile_template(template: str) -> list[Text | Field]: @@ -334,7 +386,7 @@ def _check_field(field: Field, allowed: frozenset[str]) -> str | None: if not _PATH_RE.fullmatch(path): return f"invalid field reference {path!r} (use dotted names such as 'device.label')" namespace = path.split(".", 1)[0] - if namespace not in _NAMESPACES or (namespace != "attributes" and path not in _KNOWN_PATHS): + if namespace not in _NAMESPACES or (namespace not in {"attributes", "inputs"} and path not in _KNOWN_PATHS): return f"unknown field {path!r}" if namespace not in allowed: return f"field {path!r} is not available for this entry" diff --git a/src/videoipath_automation_tool/blueprints/processors/__init__.py b/src/videoipath_automation_tool/provisioning/processors/__init__.py similarity index 94% rename from src/videoipath_automation_tool/blueprints/processors/__init__.py rename to src/videoipath_automation_tool/provisioning/processors/__init__.py index 8b9f3ad..daebdc5 100644 --- a/src/videoipath_automation_tool/blueprints/processors/__init__.py +++ b/src/videoipath_automation_tool/provisioning/processors/__init__.py @@ -3,7 +3,7 @@ A processor *decides* configuration; it never writes. It receives a detached, read-only :class:`ProcessingContext` (exact device/module scope, immutable records built from fresh server reads — no reference to the live Inspect snapshot) and validated parameters, and returns a typed -:class:`~videoipath_automation_tool.blueprints.models.ProcessorResult`. The engine validates the +:class:`~videoipath_automation_tool.provisioning.models.ProcessorResult`. The engine validates the output (scope, conflicts, capabilities), renders names, compares, and performs the writes. Custom processors are trusted Python code, not sandboxed: the contract prevents accidental @@ -21,8 +21,17 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError -from videoipath_automation_tool.blueprints.errors import BlueprintValidationError, ProcessorInputError, ValidationIssue -from videoipath_automation_tool.blueprints.models import InterfaceBinding, ProcessorResult, issues_from_pydantic +from videoipath_automation_tool.provisioning.errors import ( + ProcessorInputError, + ProvisioningValidationError, + ValidationIssue, +) +from videoipath_automation_tool.provisioning.models import ( + InterfaceBinding, + PortBinding, + ProcessorResult, + issues_from_pydantic, +) ParamsT = TypeVar("ParamsT", bound=BaseModel) @@ -129,6 +138,7 @@ class ProcessingContext(_Record): ports: tuple[PortRecord, ...] = () vertices: tuple[VertexRecord, ...] = () interfaces: tuple[InterfaceBinding, ...] = () + port_bindings: tuple[PortBinding, ...] = () owner: DriverContext | None = None inventory: DriverContext | None = None capabilities: frozenset[str] = frozenset() @@ -239,7 +249,7 @@ def register(self, processor_id: str, processor_cls: type[VertexProcessor[Any]]) def get(self, processor_id: str) -> type[VertexProcessor[Any]]: processor_cls = self._processors.get(processor_id) if processor_cls is None: - raise BlueprintValidationError( + raise ProvisioningValidationError( ValidationIssue( message=f"Unknown processor '{processor_id}'. Available: {', '.join(self.ids()) or 'none'}.", code="processor.unknown", @@ -268,7 +278,7 @@ def validate_params(self, processor_id: str, params: Mapping[str, Any], path: tu try: return params_model.model_validate(data) except ValidationError as exc: - raise BlueprintValidationError(issues_from_pydantic(exc, data=data, prefix=path)) from None + raise ProvisioningValidationError(issues_from_pydantic(exc, data=data, prefix=path)) from None def __repr__(self) -> str: return f"ProcessorRegistry({', '.join(self.ids())})" @@ -280,7 +290,7 @@ def __repr__(self) -> str: def _builtin_processors() -> dict[str, type[VertexProcessor[Any]]]: - from videoipath_automation_tool.blueprints.processors.matrox import MatroxConvertIPProcessor + from videoipath_automation_tool.provisioning.processors.matrox import MatroxConvertIPProcessor return {"matrox.convertip.default": MatroxConvertIPProcessor} diff --git a/src/videoipath_automation_tool/blueprints/processors/matrox.py b/src/videoipath_automation_tool/provisioning/processors/matrox.py similarity index 97% rename from src/videoipath_automation_tool/blueprints/processors/matrox.py rename to src/videoipath_automation_tool/provisioning/processors/matrox.py index fab1354..7332dc7 100644 --- a/src/videoipath_automation_tool/blueprints/processors/matrox.py +++ b/src/videoipath_automation_tool/provisioning/processors/matrox.py @@ -27,12 +27,12 @@ from pydantic import BaseModel, ConfigDict -from videoipath_automation_tool.blueprints.errors import ( - BlueprintCapabilityError, +from videoipath_automation_tool.provisioning.errors import ( ProcessorInputError, + ProvisioningCapabilityError, TopologyNotReadyError, ) -from videoipath_automation_tool.blueprints.models import ( +from videoipath_automation_tool.provisioning.models import ( DevicePatch, Diagnostic, EndpointIdentity, @@ -41,7 +41,7 @@ VertexEdit, VertexPatch, ) -from videoipath_automation_tool.blueprints.processors import ProcessingContext, VertexProcessor, VertexRecord +from videoipath_automation_tool.provisioning.processors import ProcessingContext, VertexProcessor, VertexRecord Layout = Literal["tx", "rx", "four_split"] @@ -68,7 +68,7 @@ class MatroxConvertIPProcessor(VertexProcessor[MatroxConvertIPParams]): def process(self, context: ProcessingContext, params: MatroxConvertIPParams) -> ProcessorResult: if params.configure_control: - raise BlueprintCapabilityError( + raise ProvisioningCapabilityError( "Matrox ConvertIP: configuring vertex control is not supported until its Inspect mapping is verified; " "remove 'configure_control' to preserve the current control settings." ) diff --git a/src/videoipath_automation_tool/provisioning/resolution.py b/src/videoipath_automation_tool/provisioning/resolution.py new file mode 100644 index 0000000..96a16d8 --- /dev/null +++ b/src/videoipath_automation_tool/provisioning/resolution.py @@ -0,0 +1,601 @@ +"""Resolve ordered v2 overlays, typed inputs, driver settings and processor parameters.""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.resources +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError + +from videoipath_automation_tool.apps.inventory.model.drivers import ( + DRIVER_ID_TO_CUSTOM_SETTINGS, + SELECTED_SCHEMA_VERSION, +) +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue +from videoipath_automation_tool.provisioning.inputs import bind_inputs, is_expression +from videoipath_automation_tool.provisioning.models import ( + Blueprint, + InventoryConfig, + InventorySettings, + Scope, + TopologyConfig, + issues_from_pydantic, +) +from videoipath_automation_tool.provisioning.naming import NamingScheme, validate_naming_inputs +from videoipath_automation_tool.provisioning.processors import ProcessorRegistry, VertexProcessor + +Provenance = dict[str, str] + + +class ResolvedInventory(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + driver_id: str + schema_version: str = SELECTED_SCHEMA_VERSION + config: InventoryConfig + custom_settings: dict[str, Any] + provenance: Provenance + + +class ResolvedTopology(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + config: TopologyConfig + processor_id: str | None = None + processor_cls: type[VertexProcessor] | None = None # type: ignore[type-arg] + params: BaseModel | None = None + provenance: Provenance + inputs: dict[str, JsonValue] = Field(default_factory=dict) + + +class ResolvedBlueprint(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + inventory: ResolvedInventory | None = None + topology: ResolvedTopology | None = None + variants: tuple[str, ...] = () + inputs: dict[str, JsonValue] = Field(default_factory=dict) + naming: NamingScheme + skipped: dict[str, str] + digest: str + + +def resolve_blueprint( + blueprint: Blueprint, + *, + scope: Scope = "all", + variants: Sequence[str] = (), + inputs: Mapping[str, JsonValue] | None = None, + registry: ProcessorRegistry | None = None, + overrides: InventorySettings | None = None, + require_processor: bool = True, +) -> ResolvedBlueprint: + """Merge and bind before any server access. Source documents are never mutated.""" + if scope not in ("all", "inventory", "topology"): + raise ProvisioningValidationError("Invalid scope; expected 'all', 'inventory', or 'topology'.") + selected = _selected_variants(blueprint, variants) + values = bind_inputs(blueprint.inputs, inputs) + document = blueprint.model_dump(mode="python", by_alias=True, exclude_unset=True) + raw = copy.deepcopy(document["defaults"]) + origins = {section: _leaf_provenance(config, "defaults") for section, config in raw.items()} + for variant in selected: + for section, patch in document.get("variants", {}).get(variant, {}).items(): + provenance = origins.setdefault(section, {}) + base = raw.get(section, {}) + patch = copy.deepcopy(patch) + source = f"variant:{variant}" + if section == "inventory" and "driver_id" in patch and patch["driver_id"] != base.get("driver_id"): + base.pop("custom_settings", None) + _forget(provenance, "custom_settings") + if section == "topology" and "vertex_processor" in patch: + base["vertex_processor"] = _merge_processor( + base.get("vertex_processor"), patch.pop("vertex_processor"), provenance, source + ) + raw[section] = _merge(base, patch, (), provenance, source) + + skipped: dict[str, str] = {} + configs: dict[str, dict[str, Any]] = {} + for section in ("inventory", "topology"): + if scope not in ("all", section): + skipped[section] = f"not in scope '{scope}'" + elif section not in raw: + if scope == section: + raise ProvisioningValidationError( + _issue( + blueprint, + ("defaults", section), + f"scope='{section}' requested but the blueprint has no '{section}' section.", + "section.missing", + ) + ) + skipped[section] = "section absent from blueprint" + else: + configs[section] = _materialize(raw[section], values, blueprint, (section,), origins[section]) + + if overrides is not None and "inventory" in configs: + configs["inventory"] = _merge( + configs["inventory"], overrides.model_dump(exclude_unset=True), (), origins["inventory"], "instance" + ) + inventory = ( + _resolve_inventory(configs["inventory"], blueprint, origins["inventory"]) if "inventory" in configs else None + ) + topology = ( + _resolve_topology(configs["topology"], blueprint, origins["topology"], registry, require_processor, values) + if "topology" in configs + else None + ) + naming = NamingScheme.layered( + inventory.config.naming.to_scheme() if inventory and inventory.config.naming else None, + topology.config.naming.to_scheme() if topology and topology.config.naming else None, + ) + for section, config in (("inventory", inventory), ("topology", topology)): + if config is not None and config.config.naming is not None: + try: + validate_naming_inputs(config.config.naming.to_scheme(), blueprint.inputs, values) + except ProvisioningValidationError as exc: + raise ProvisioningValidationError( + [_relocate_relative(blueprint, (section,), issue, config.provenance) for issue in exc.issues] + ) from None + return ResolvedBlueprint( + inventory=inventory, + topology=topology, + variants=selected, + inputs=values, + naming=naming, + skipped=skipped, + digest=blueprint_digest(blueprint), + ) + + +def validate_all_variants( + blueprint: Blueprint, + registry: ProcessorRegistry | None, + *, + inputs: Mapping[str, JsonValue] | None = None, + variants: Sequence[str] | None = None, +) -> None: + """Check the base and each overlay, or one explicitly selected combination, offline.""" + combinations = [variants] if variants is not None else [(), *((name,) for name in blueprint.variant_names())] + issues: list[ValidationIssue] = [] + for combination in combinations: + try: + selected = _selected_variants(blueprint, combination) + except ProvisioningValidationError as exc: + issues.extend(exc.issues) + continue + sections = set(blueprint.defaults.model_fields_set) + for name in selected: + sections.update(blueprint.variants[name].model_fields_set) + # Validate both sections independently so a broken Inventory setting does not hide + # a Topology or processor error in the same configuration. + for section in ("inventory", "topology"): + if section not in sections: + continue + try: + resolved = resolve_blueprint( + blueprint, + scope=section, + variants=selected, + inputs=inputs, + registry=registry, + require_processor=False, + ) + except ProvisioningValidationError as exc: + issues.extend(exc.issues) + continue + topology = resolved.topology + if ( + registry is not None + and topology is not None + and topology.processor_id is not None + and topology.processor_cls is None + ): + path = _source_path( + ("topology",), + ("vertex_processor", "processor_type"), + topology.provenance, + "vertex_processor.processor_type", + ) + issues.append( + _issue( + blueprint, + path, + f"Processor '{topology.processor_id}' is not registered; its parameters were not validated.", + "processor.unavailable", + ) + ) + if issues: + unique = {(issue.path, issue.code, issue.message): issue for issue in issues} + raise ProvisioningValidationError(list(unique.values())) + + +def blueprint_digest(blueprint: Blueprint) -> str: + document = blueprint.model_dump(mode="json", by_alias=True, exclude_unset=True) + return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def blueprint_json_schema( + registry: ProcessorRegistry | None = None, *, restrict_processor_types: bool = False +) -> dict[str, Any]: + """Draft 2020-12 document schema; typed value positions also accept input references.""" + if restrict_processor_types and registry is None: + raise ValueError("restrict_processor_types requires a registry.") + schema = Blueprint.model_json_schema(by_alias=True) + schema.update( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", + "title": "VideoIPath Automation Tool blueprint (schema_version 1)", + } + ) + configuration = schema["$defs"]["BlueprintConfiguration"] + # Section keys are structural and explicit null is rejected by the loader. + for section in ("inventory", "topology"): + configuration["properties"][section] = {"$ref": f"#/$defs/{section.title()}ConfigTemplate"} + schema["properties"]["defaults"]["anyOf"] = [{"required": ["inventory"]}, {"required": ["topology"]}] + if registry is None: + return schema + defs = schema["$defs"] + branches: list[dict[str, Any]] = [] + for processor_id in registry.ids(): + params = registry.get(processor_id).params_model.model_json_schema( + by_alias=True, ref_template="#/$defs/{model}" + ) + extra_defs = params.pop("$defs", {}) + # Prefix custom parameter definitions to avoid overwriting document definitions. + prefix = processor_id + "." + + def rewrite(value: Any, prefix: str = prefix) -> Any: + if isinstance(value, list): + return [rewrite(item) for item in value] + if not isinstance(value, dict): + return value + result = {key: rewrite(item) for key, item in value.items()} + if "$ref" in result and result["$ref"].startswith("#/$defs/"): + result["$ref"] = "#/$defs/" + prefix + result["$ref"].removeprefix("#/$defs/") + if "properties" in result: + result["properties"] = {key: _value_schema(item) for key, item in result["properties"].items()} + if "items" in result: + result["items"] = _value_schema(result["items"]) + if isinstance(result.get("additionalProperties"), dict): + result["additionalProperties"] = _value_schema(result["additionalProperties"]) + return result + + defs.update({prefix + name: rewrite(value) for name, value in extra_defs.items()}) + branches.append( + { + "if": {"properties": {"processor_type": {"const": processor_id}}, "required": ["processor_type"]}, + "then": {"properties": {"params": _value_schema(rewrite(params))}}, + } + ) + spec = defs["VertexProcessorSpecTemplate"] + if branches: + spec["allOf"] = branches + if restrict_processor_types: + spec["properties"]["processor_type"] = {"type": "string", "enum": registry.ids()} + return schema + + +def published_json_schema() -> dict[str, Any]: + resource = importlib.resources.files("videoipath_automation_tool.provisioning").joinpath( + "schemas/blueprint-v1.schema.json" + ) + return json.loads(resource.read_text(encoding="utf-8")) + + +# --- Internal --- + + +def _resolve_inventory(data: dict[str, Any], blueprint: Blueprint, provenance: Provenance) -> ResolvedInventory: + entry_path = ("inventory",) + config = _validate_entry(InventoryConfig, data, blueprint, entry_path, provenance) + if config.driver_id is None: + raise ProvisioningValidationError( + _issue( + blueprint, + _source_path(entry_path, ("driver_id",), provenance, "driver_id"), + "Field required (inventory needs a driver_id).", + "missing", + ) + ) + model = DRIVER_ID_TO_CUSTOM_SETTINGS.get(config.driver_id) + if model is None: + raise ProvisioningValidationError( + _issue( + blueprint, + _source_path(entry_path, ("driver_id",), provenance, "driver_id"), + f"Unknown driver '{config.driver_id}' for the selected driver schema {SELECTED_SCHEMA_VERSION}.", + "driver.unknown", + ) + ) + custom = dict(config.custom_settings or {}) + return ResolvedInventory( + driver_id=config.driver_id, + config=config, + custom_settings=_validate_custom_settings(model, custom, blueprint, entry_path, provenance, config.driver_id), + provenance=provenance, + ) + + +def _resolve_topology( + data: dict[str, Any], + blueprint: Blueprint, + provenance: Provenance, + registry: ProcessorRegistry | None, + require_processor: bool, + inputs: dict[str, JsonValue], +) -> ResolvedTopology: + entry_path = ("topology",) + config = _validate_entry(TopologyConfig, data, blueprint, entry_path, provenance) + spec = config.vertex_processor + common = {"config": config, "provenance": provenance, "inputs": copy.deepcopy(inputs)} + if spec is None: + return ResolvedTopology(**common) + type_path = _source_path( + entry_path, ("vertex_processor", "processor_type"), provenance, "vertex_processor.processor_type" + ) + if spec.processor_type is None: + raise ProvisioningValidationError(_issue(blueprint, type_path, "Field required.", "missing")) + if registry is None or (spec.processor_type not in registry and not require_processor): + return ResolvedTopology(processor_id=spec.processor_type, **common) + try: + processor_cls = registry.get(spec.processor_type) + except ProvisioningValidationError as exc: + raise ProvisioningValidationError( + _issue(blueprint, type_path, exc.issues[0].message, "processor.unknown") + ) from None + try: + params = registry.validate_params(spec.processor_type, spec.params) + except ProvisioningValidationError as exc: + raise ProvisioningValidationError( + [ + _relocate_relative( + blueprint, + entry_path, + issue.model_copy(update={"path": f"vertex_processor.params.{issue.path}".rstrip(".")}), + provenance, + ) + for issue in exc.issues + ] + ) from None + return ResolvedTopology(processor_id=spec.processor_type, processor_cls=processor_cls, params=params, **common) + + +def _selected_variants(blueprint: Blueprint, variants: Sequence[str]) -> tuple[str, ...]: + if ( + isinstance(variants, (str, bytes)) + or not isinstance(variants, Sequence) + or any(not isinstance(name, str) for name in variants) + ): + raise TypeError("variants must be a sequence of names, not a string.") + selected = tuple(variants) + if len(set(selected)) != len(selected): + raise ProvisioningValidationError( + ValidationIssue(path="variants", message="A variant may only be selected once.", code="variant.duplicate") + ) + unknown = [name for name in selected if name not in blueprint.variants] + if unknown: + raise ProvisioningValidationError( + ValidationIssue( + path="variants", + message=f"Unknown variant '{unknown[0]}'. Available: {', '.join(blueprint.variant_names()) or 'none'}.", + code="variant.unknown", + ) + ) + return selected + + +def _materialize( + value: Any, + inputs: dict[str, JsonValue], + blueprint: Blueprint, + entry_path: tuple[str, ...], + provenance: Provenance, + path: tuple[str | int, ...] = (), +) -> Any: + if is_expression(value): + key = ".".join(map(str, path)) + origin = _origin(provenance, key) + if "$literal" in value: + result = copy.deepcopy(value["$literal"]) + _forget(provenance, key) + provenance.update(_leaf_provenance(result, origin, path)) + return result + name = value["$input"] + if name not in inputs: + raise ProvisioningValidationError( + _issue( + blueprint, + _source_path(entry_path, path, provenance, key), + "Required input is missing.", + "input.missing", + ) + ) + result = copy.deepcopy(inputs[name]) + _forget(provenance, key) + provenance.update(_leaf_provenance(result, f"{origin}|input:{name}", path)) + return result + if isinstance(value, dict): + return { + key: _materialize(child, inputs, blueprint, entry_path, provenance, path + (key,)) + for key, child in value.items() + } + if isinstance(value, list): + return [ + _materialize(child, inputs, blueprint, entry_path, provenance, path + (index,)) + for index, child in enumerate(value) + ] + return value + + +def _merge( + base: dict[str, Any], patch: dict[str, Any], path: tuple[str, ...], provenance: Provenance, source: str +) -> dict[str, Any]: + merged = dict(base) + for key, value in patch.items(): + child, current = path + (key,), merged.get(key) + if ( + isinstance(value, dict) + and isinstance(current, dict) + and not _atomic(child) + and not is_expression(value) + and not is_expression(current) + ): + merged[key] = _merge(current, value, child, provenance, source) + else: + merged[key] = copy.deepcopy(value) + _forget(provenance, ".".join(child)) + provenance.update(_leaf_provenance(value, source, child)) + return merged + + +def _merge_processor(base: Any, patch: Any, provenance: Provenance, source: str) -> Any: + if ( + patch is not None + and isinstance(base, dict) + and isinstance(patch, dict) + and patch.get("processor_type", base.get("processor_type")) == base.get("processor_type") + ): + return _merge(base, patch, ("vertex_processor",), provenance, source) + _forget(provenance, "vertex_processor") + provenance.update(_leaf_provenance(patch, source, ("vertex_processor",))) + return copy.deepcopy(patch) + + +def _atomic(path: tuple[str | int, ...]) -> bool: + return bool(path) and ( + (len(path) == 2 and path[0] == "naming") + or path[-1] == "tags" + or isinstance(path[-1], str) + and path[-1].endswith("_tags") + ) + + +def _leaf_provenance(value: Any, source: str, path: tuple[str | int, ...] = ()) -> Provenance: + if isinstance(value, dict) and value and not is_expression(value) and not _atomic(path): + return { + key: origin + for name, child in value.items() + for key, origin in _leaf_provenance(child, source, path + (name,)).items() + } + return {".".join(map(str, path)): source} if path else {} + + +def _forget(provenance: Provenance, prefix: str) -> None: + for stale in [key for key in provenance if key == prefix or key.startswith(prefix + ".")]: + del provenance[stale] + + +def _origin(provenance: Provenance, key: str) -> str: + return provenance.get(key) or next( + (source for path, source in provenance.items() if path.startswith(key + ".") or key.startswith(path + ".")), + "defaults", + ) + + +def _source_path( + entry_path: tuple[str, ...], relative: tuple[Any, ...], provenance: Provenance, key: str +) -> tuple[Any, ...]: + source = _origin(provenance, key).split("|input:", 1)[0] + section = entry_path[0] + if source == "instance": + return ("device", "inventory_overrides") + relative + if source.startswith("variant:"): + return ("variants", source.removeprefix("variant:"), section) + relative + return ("defaults", section) + relative + + +def _value_schema(schema: dict[str, Any]) -> dict[str, Any]: + return {"anyOf": [schema, {"$ref": "#/$defs/InputReference"}, {"$ref": "#/$defs/LiteralValue"}]} + + +def _validate_entry( + model: type[BaseModel], + data: dict[str, Any], + blueprint: Blueprint, + entry_path: tuple[str, ...], + provenance: Provenance, +) -> Any: + try: + return model.model_validate(data) + except ValidationError as exc: + issues = issues_from_pydantic(exc, data=data) + raise ProvisioningValidationError( + [_relocate_relative(blueprint, entry_path, issue, provenance) for issue in issues] + ) from None + + +def _validate_custom_settings( + model: type[BaseModel], + custom: dict[str, Any], + blueprint: Blueprint, + entry_path: tuple[str, ...], + provenance: Provenance, + driver_id: str, +) -> dict[str, Any]: + allowed = set(model.model_fields) - {"driver_id"} + unknown = sorted(set(custom) - allowed) + if unknown: + raise ProvisioningValidationError( + [ + _issue( + blueprint, + _source_path(entry_path, ("custom_settings", key), provenance, f"custom_settings.{key}"), + f"Unknown custom setting for driver '{driver_id}' (schema {SELECTED_SCHEMA_VERSION}). " + f"Allowed: {', '.join(sorted(allowed)) or 'none'}.", + "driver.unknown_setting", + ) + for key in unknown + ] + ) + try: + instance = model.model_validate(custom, strict=True, by_name=True, by_alias=False) + except ValidationError as exc: + issues = issues_from_pydantic(exc, data=custom) + raise ProvisioningValidationError( + [ + _relocate( + blueprint, + _source_path(entry_path, ("custom_settings",), provenance, f"custom_settings.{issue.path}"), + issue.model_copy( + update={"message": f"{issue.message} (driver '{driver_id}', schema {SELECTED_SCHEMA_VERSION})"} + ), + ) + for issue in issues + ] + ) from None + return {key: getattr(instance, key) for key in custom} + + +def _relocate_relative( + blueprint: Blueprint, entry_path: tuple[str, ...], issue: ValidationIssue, provenance: Provenance +) -> ValidationIssue: + relative = tuple(issue.path.split(".")) if issue.path else () + path = _source_path(entry_path, relative, provenance, issue.path) + return _issue(blueprint, path, issue.message, issue.code) + + +def _relocate(blueprint: Blueprint, prefix: tuple[Any, ...], issue: ValidationIssue) -> ValidationIssue: + relative = tuple(issue.path.split(".")) if issue.path else () + return _issue(blueprint, prefix + relative, issue.message, issue.code) + + +def _issue(blueprint: Blueprint, path: tuple[Any, ...], message: str, code: str) -> ValidationIssue: + keys = tuple(int(part) if isinstance(part, str) and part.isdigit() else part for part in path) + source, line, column = None, None, None + for end in range(len(keys), -1, -1): + source, line, column = blueprint.location_of(keys[:end]) + if line is not None: + break + return ValidationIssue( + path=".".join(str(part) for part in path), + message=message, + code=code, + source=source, + line=line, + column=column, + ) diff --git a/src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json b/src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json new file mode 100644 index 0000000..34179e2 --- /dev/null +++ b/src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json @@ -0,0 +1,2142 @@ +{ + "$defs": { + "BlueprintConfiguration": { + "additionalProperties": false, + "description": "A configuration or overlay. Section keys are structural, not input references.", + "properties": { + "inventory": { + "$ref": "#/$defs/InventoryConfigTemplate" + }, + "topology": { + "$ref": "#/$defs/TopologyConfigTemplate" + } + }, + "title": "BlueprintConfiguration", + "type": "object" + }, + "BlueprintNamingTemplate": { + "additionalProperties": false, + "properties": { + "device_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Device Description" + }, + "device_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Device Label" + }, + "endpoint_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Endpoint Description" + }, + "endpoint_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Endpoint Label" + }, + "inventory_description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory Description" + }, + "inventory_label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory Label" + } + }, + "title": "BlueprintNamingTemplate", + "type": "object" + }, + "CatalogIdTemplate": { + "additionalProperties": false, + "properties": { + "id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Id" + } + }, + "required": [ + "id" + ], + "title": "CatalogIdTemplate", + "type": "object" + }, + "CoordinatesTemplate": { + "additionalProperties": false, + "properties": { + "x": { + "anyOf": [ + { + "type": "number" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "X" + }, + "y": { + "anyOf": [ + { + "type": "number" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Y" + } + }, + "required": [ + "x", + "y" + ], + "title": "CoordinatesTemplate", + "type": "object" + }, + "DevicePatchTemplate": { + "additionalProperties": false, + "properties": { + "coordinates": { + "anyOf": [ + { + "$ref": "#/$defs/CoordinatesTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Coordinates" + }, + "icon_size": { + "anyOf": [ + { + "enum": [ + "auto", + "large", + "medium", + "small" + ], + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Icon Size" + }, + "icon_type": { + "anyOf": [ + { + "enum": [ + "default", + "none", + "device", + "camera", + "monitor", + "encoder", + "decoder", + "audioMixer", + "videoMixer", + "processingDevice", + "transportStreamProcessor", + "mediaDevice", + "server", + "gateway", + "ipSwitchRouter", + "vlanCloud", + "videoAudioRouterMatrix", + "encoderDecoder" + ], + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Icon Type" + }, + "sdp_strategy": { + "anyOf": [ + { + "enum": [ + "always", + "once", + "video" + ], + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sdp Strategy" + }, + "site_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Site Id" + }, + "tags": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDeltaTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "title": "DevicePatchTemplate", + "type": "object" + }, + "FieldTemplate": { + "additionalProperties": false, + "properties": { + "field": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Field" + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Format" + }, + "mapping": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mapping" + }, + "prefix": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "default": "", + "title": "Prefix" + }, + "suffix": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "default": "", + "title": "Suffix" + } + }, + "required": [ + "field" + ], + "title": "FieldTemplate", + "type": "object" + }, + "GenericSettingsTemplate": { + "additionalProperties": false, + "properties": { + "enable_https": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enable Https" + }, + "http_auth": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Http Auth" + }, + "trust_all_certificates": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Trust All Certificates" + } + }, + "title": "GenericSettingsTemplate", + "type": "object" + }, + "InputDefinition": { + "additionalProperties": false, + "description": "A small JSON Schema vocabulary; defaults are values, never expressions.", + "properties": { + "additionalProperties": { + "default": false, + "title": "Additionalproperties", + "type": "boolean" + }, + "default": { + "$ref": "#/$defs/JsonValue", + "default": null + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "enum": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/JsonValue" + }, + "minItems": 1, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enum" + }, + "items": { + "anyOf": [ + { + "$ref": "#/$defs/InputDefinition" + }, + { + "type": "null" + } + ], + "default": null + }, + "properties": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/$defs/InputDefinition" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Properties" + }, + "required": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Required" + }, + "type": { + "enum": [ + "string", + "integer", + "number", + "boolean", + "array", + "object", + "null" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InputDefinition", + "type": "object" + }, + "InputReference": { + "additionalProperties": false, + "properties": { + "$input": { + "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", + "title": "$Input", + "type": "string" + } + }, + "required": [ + "$input" + ], + "title": "InputReference", + "type": "object" + }, + "InventoryConfigTemplate": { + "additionalProperties": false, + "properties": { + "active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Active" + }, + "custom_settings": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + {}, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Custom Settings" + }, + "driver_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Driver Id" + }, + "generic_settings": { + "anyOf": [ + { + "$ref": "#/$defs/GenericSettingsTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Generic Settings" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "propertyNames": { + "minLength": 1 + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Metadata" + }, + "naming": { + "anyOf": [ + { + "$ref": "#/$defs/BlueprintNamingTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Naming" + }, + "snmp": { + "anyOf": [ + { + "$ref": "#/$defs/SnmpSelectionTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Snmp" + } + }, + "title": "InventoryConfigTemplate", + "type": "object" + }, + "JoinTemplate": { + "additionalProperties": false, + "properties": { + "join": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/TextTemplate" + }, + { + "$ref": "#/$defs/FieldTemplate" + }, + { + "$ref": "#/$defs/JoinTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Join" + }, + "separator": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "default": "", + "title": "Separator" + }, + "skip_missing": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "default": false, + "title": "Skip Missing" + } + }, + "required": [ + "join" + ], + "title": "JoinTemplate", + "type": "object" + }, + "JsonValue": {}, + "LiteralValue": { + "additionalProperties": false, + "properties": { + "$literal": { + "$ref": "#/$defs/JsonValue" + } + }, + "required": [ + "$literal" + ], + "title": "LiteralValue", + "type": "object" + }, + "ModulePatchTemplate": { + "additionalProperties": false, + "properties": { + "tags": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDeltaTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "title": "ModulePatchTemplate", + "type": "object" + }, + "PortSelectorTemplate": { + "additionalProperties": false, + "properties": { + "factory_label": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Factory Label" + }, + "kind": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Kind" + }, + "port_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Port Id" + } + }, + "title": "PortSelectorTemplate", + "type": "object" + }, + "SnmpSelectionTemplate": { + "additionalProperties": false, + "properties": { + "configuration": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/CatalogIdTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Configuration" + }, + "use_global_settings": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use Global Settings" + } + }, + "title": "SnmpSelectionTemplate", + "type": "object" + }, + "TagDeltaTemplate": { + "additionalProperties": false, + "properties": { + "add": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Add" + }, + "remove": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Remove" + } + }, + "title": "TagDeltaTemplate", + "type": "object" + }, + "TextTemplate": { + "additionalProperties": false, + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Text" + } + }, + "required": [ + "text" + ], + "title": "TextTemplate", + "type": "object" + }, + "TopologyConfigTemplate": { + "additionalProperties": false, + "properties": { + "device": { + "anyOf": [ + { + "$ref": "#/$defs/DevicePatchTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Device" + }, + "ip_vertex_mapping": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "minItems": 1, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "propertyNames": { + "minLength": 1 + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ip Vertex Mapping" + }, + "module": { + "anyOf": [ + { + "$ref": "#/$defs/ModulePatchTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Module" + }, + "naming": { + "anyOf": [ + { + "$ref": "#/$defs/BlueprintNamingTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Naming" + }, + "port_mapping": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/PortSelectorTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "minItems": 1, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "propertyNames": { + "minLength": 1 + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Port Mapping" + }, + "vertex_processor": { + "anyOf": [ + { + "$ref": "#/$defs/VertexProcessorSpecTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vertex Processor" + }, + "vertices": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/VertexOverrideTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vertices" + } + }, + "title": "TopologyConfigTemplate", + "type": "object" + }, + "VertexOverrideTemplate": { + "additionalProperties": false, + "properties": { + "direction": { + "anyOf": [ + { + "enum": [ + "In", + "Out", + "Internal", + "Undecided" + ], + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Direction" + }, + "factory_label": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Factory Label" + }, + "fields": { + "anyOf": [ + { + "$ref": "#/$defs/VertexPatchTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Fields" + }, + "kind": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Kind" + }, + "vertex_id": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vertex Id" + } + }, + "required": [ + "fields" + ], + "title": "VertexOverrideTemplate", + "type": "object" + }, + "VertexPatchTemplate": { + "additionalProperties": false, + "properties": { + "active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Active" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Label" + }, + "main_destination_port": { + "anyOf": [ + { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Main Destination Port" + }, + "sdp_support": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sdp Support" + }, + "sips_mode": { + "anyOf": [ + { + "enum": [ + "NONE", + "SIPSAuto", + "SIPSDuplicate", + "SIPSMerge", + "SIPSSplit" + ], + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sips Mode" + }, + "spare_destination_port": { + "anyOf": [ + { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spare Destination Port" + }, + "supports_static_igmp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Supports Static Igmp" + }, + "tags": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/TagDeltaTemplate" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "use_as_endpoint": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Use As Endpoint" + } + }, + "title": "VertexPatchTemplate", + "type": "object" + }, + "VertexProcessorSpecTemplate": { + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "processor_type": { + "const": "matrox.convertip.default" + } + }, + "required": [ + "processor_type" + ] + }, + "then": { + "properties": { + "params": { + "anyOf": [ + { + "additionalProperties": false, + "description": "Parameters. Omitted tag parameters leave local tags unchanged (``[]`` clears them); an omitted\n``redundant_streams`` preserves the SIPS mode (``true`` \u2192 ``SIPSAuto``, ``false`` \u2192 ``NONE``).", + "properties": { + "allow_null_audio_label": { + "anyOf": [ + { + "default": false, + "title": "Allow Null Audio Label", + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "audio_receiver_tags": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/matrox.convertip.default.TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Receiver Tags" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "audio_sender_tags": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/matrox.convertip.default.TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Sender Tags" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "configure_control": { + "anyOf": [ + { + "default": false, + "title": "Configure Control", + "type": "boolean" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "mode": { + "anyOf": [ + { + "default": "auto", + "enum": [ + "auto", + "tx", + "rx", + "four_split" + ], + "title": "Mode", + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "redundant_streams": { + "anyOf": [ + { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Redundant Streams" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "video_receiver_tags": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/matrox.convertip.default.TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Video Receiver Tags" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "video_sender_tags": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "array" + }, + { + "$ref": "#/$defs/matrox.convertip.default.TagDelta" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Video Sender Tags" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + } + }, + "title": "MatroxConvertIPParams", + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + } + } + } + } + ], + "properties": { + "params": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + {}, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "type": "object" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ], + "title": "Params" + }, + "processor_type": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Processor Type" + } + }, + "title": "VertexProcessorSpecTemplate", + "type": "object" + }, + "matrox.convertip.default.TagDelta": { + "additionalProperties": false, + "description": "Additive tag change that preserves unrelated local tags. Tag references are exact catalog\ntag ids (for example ``Format~~V_1080p50``); the engine never creates catalog entries.", + "properties": { + "add": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "title": "Add", + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "remove": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + }, + "title": "Remove", + "type": "array" + }, + { + "$ref": "#/$defs/InputReference" + }, + { + "$ref": "#/$defs/LiteralValue" + } + ] + } + }, + "title": "TagDelta", + "type": "object" + } + }, + "$id": "https://github.com/SWR-MoIP/VideoIPath-Automation-Tool/blueprint-v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "A reusable configuration with typed inputs and ordered variant overlays.\n\nLoad with :meth:`load` (file), :meth:`from_yaml` (text/bytes), or :meth:`from_dict` (mapping).\nLoading never connects to a server. Variants patch ``defaults`` in caller-selected order.", + "properties": { + "defaults": { + "$ref": "#/$defs/BlueprintConfiguration", + "anyOf": [ + { + "required": [ + "inventory" + ] + }, + { + "required": [ + "topology" + ] + } + ] + }, + "inputs": { + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_-]*$": { + "$ref": "#/$defs/InputDefinition" + } + }, + "title": "Inputs", + "type": "object" + }, + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "variants": { + "patternProperties": { + "^[A-Za-z0-9][A-Za-z0-9_.-]*$": { + "$ref": "#/$defs/BlueprintConfiguration" + } + }, + "title": "Variants", + "type": "object" + } + }, + "required": [ + "schema_version", + "defaults" + ], + "title": "VideoIPath Automation Tool blueprint (schema_version 1)", + "type": "object" +} diff --git a/src/videoipath_automation_tool/provisioning/templates.py b/src/videoipath_automation_tool/provisioning/templates.py new file mode 100644 index 0000000..5d19547 --- /dev/null +++ b/src/videoipath_automation_tool/provisioning/templates.py @@ -0,0 +1,147 @@ +"""Derive strict document templates from the same models used after input resolution. + +Value positions accept references and literal escapes. Discriminator ids stay static. Keeping +this derivation centralized makes the editor schema and the loader follow the runtime fields. +""" + +from __future__ import annotations + +import copy +import types +from functools import reduce +from operator import or_ +from typing import Annotated, Any, ForwardRef, Union, get_args, get_origin + +from pydantic import BaseModel, ConfigDict, create_model, model_validator + +from videoipath_automation_tool.provisioning.errors import ProvisioningValidationError, ValidationIssue +from videoipath_automation_tool.provisioning.inputs import ( + InputDefinition, + InputReference, + LiteralValue, + contains_expression, + is_expression, +) +from videoipath_automation_tool.provisioning.naming import reference_paths + + +def template_models(*models: type[BaseModel]) -> list[type[BaseModel]]: + """Generate recursive template types, preserving constraints and literal-only validators.""" + cache: dict[type[BaseModel], type[BaseModel] | ForwardRef] = {} + + def annotation(value: Any) -> Any: + origin, args = get_origin(value), get_args(value) + if origin is Annotated: + return Annotated[annotation(args[0]), *args[1:]] + if origin in (Union, types.UnionType): + return reduce(or_, (annotation(arg) for arg in args)) + if origin is list: + return list[annotation(args[0]) | InputReference | LiteralValue] + if origin is dict: + return dict[args[0], annotation(args[1]) | InputReference | LiteralValue] + if isinstance(value, type) and issubclass(value, BaseModel): + return build(value) + return value + + def build(model: type[BaseModel]) -> type[BaseModel] | ForwardRef: + if model in cache: + return cache[model] + name = f"{model.__name__}Template" + cache[model] = ForwardRef(name) + fields: dict[str, Any] = {} + for key, original in model.model_fields.items(): + info = copy.copy(original) + value_type = annotation(original.annotation) + if original.metadata: + value_type = Annotated[value_type, *original.metadata] + info.metadata = [] + if key not in {"driver_id", "processor_type"}: + value_type = value_type | InputReference | LiteralValue + fields[key] = (value_type, info) + + @model_validator(mode="before") + def validate_literals(data: Any) -> Any: + if not contains_expression(data): + model.model_validate(data) + return data + + generated = create_model( + name, + __config__=ConfigDict(extra="forbid", strict=True, frozen=True), + __validators__={"validate_literals": validate_literals}, + **fields, + ) + cache[model] = generated + return generated + + result = [build(model) for model in models] + namespace = {value.__name__: value for value in cache.values() if isinstance(value, type)} + for generated in namespace.values(): + generated.model_rebuild(_types_namespace=namespace) + return result + + +def validate_document_references(document: dict[str, Any], definitions: dict[str, InputDefinition]) -> None: + """Check operators and declared naming paths, including in unused variants.""" + issues: list[ValidationIssue] = [] + + def problem(path: tuple[str | int, ...], message: str) -> None: + issues.append(ValidationIssue(path=".".join(map(str, path)), message=message, code="input.reference")) + + def naming_path(field: str, path: tuple[str | int, ...]) -> None: + if not field.startswith("inputs."): + return + parts = field.split(".")[1:] + definition = definitions.get(parts[0]) + if definition is None: + problem(path, f"Unknown input reference '{parts[0]}'.") + return + for part in parts[1:]: + properties = definition.properties or {} + if definition.type != "object": + problem(path, "Naming input path traverses a non-object input.") + return + if part not in properties: + if not definition.additional_properties: + problem(path, f"Unknown property in naming input reference '{field}'.") + return + definition = properties[part] + + def naming_expression(value: Any, path: tuple[str | int, ...]) -> None: + if isinstance(value, str): + try: + for field in reference_paths(value): + naming_path(field, path) + except ValueError as exc: + problem(path, str(exc)) + elif isinstance(value, dict): + if isinstance(value.get("field"), str): + naming_path(value["field"], path + ("field",)) + if isinstance(value.get("join"), list): + for index, child in enumerate(value["join"]): + naming_expression(child, path + ("join", index)) + + def walk(value: Any, path: tuple[str | int, ...]) -> None: + if is_expression(value): + if len(value) != 1: + problem(path, "An input reference or literal escape must contain exactly one operator.") + elif "$input" in value: + name = value["$input"] + if not isinstance(name, str) or name not in definitions: + problem(path, "Input reference must name a declared input.") + if path[-1] in {"driver_id", "processor_type", "vertex_processor"}: + problem(path, "Driver and processor declarations must remain static.") + return + if len(path) >= 2 and path[-2] == "naming": + naming_expression(value, path) + if isinstance(value, dict): + for key, child in value.items(): + walk(child, path + (key,)) + elif isinstance(value, list): + for index, child in enumerate(value): + walk(child, path + (index,)) + + walk(document.get("defaults", {}), ("defaults",)) + walk(document.get("variants", {}), ("variants",)) + if issues: + raise ProvisioningValidationError(issues) diff --git a/src/vipat_cli_scripts/test_runner.py b/src/vipat_cli_scripts/test_runner.py index 9a290da..3a1f409 100644 --- a/src/vipat_cli_scripts/test_runner.py +++ b/src/vipat_cli_scripts/test_runner.py @@ -9,7 +9,7 @@ from vipat_cli_scripts.project_env import prepare_e2e_env _UNIT_ARGS = ["-m", "not e2e", "--ignore=tests/e2e"] -_E2E_ARGS = ["-m", "e2e", "tests/e2e", "--no-cov"] +_E2E_ARGS = ["-m", "e2e", "-o", "testpaths=tests/e2e", "--no-cov"] def _run(args: list[str], *, extra: list[str] | None = None) -> int: diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ca345d9..007f45a 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -6,7 +6,7 @@ Version gates (by VideoIPath major year): * Topology e2e (``apps/test_topology.py``) is skipped when major > 2025. - * Inspect e2e (``apps/test_inspect.py`` and ``workflows/``) is skipped when major < 2025. + * Inspect e2e (``apps/test_inspect.py``, ``workflows/``, and ``provisioning/``) is skipped when major < 2025. E2e entry points load ``.env`` and enable the suite automatically. E2e runs never collect coverage (``--no-cov``). @@ -15,6 +15,7 @@ * ``workflows/`` — general, ordered "build the scenario step by step" suites: the generic network-builder (one suite per :mod:`networks` architecture) and the cross-app onboarding pipeline. * ``apps/`` — focused per-app suites (inventory, inspect, topology, preferences, profile, security). + * ``provisioning/`` — independent declarative plan/apply workflows using the server's mock driver. Everything the suite writes is namespaced (``E2E-`` label prefix + ``vipat-e2e`` tag) so a shared local instance is safe. Cleanup is a single session-start sweep that removes every ``E2E-`` @@ -25,9 +26,9 @@ from __future__ import annotations import os +from collections.abc import Iterator from itertools import count from pathlib import Path -from typing import Iterator, Optional import pytest @@ -51,7 +52,7 @@ def _e2e_enabled() -> bool: return os.environ.get("VIPAT_E2E_ENABLED", "").strip() == "1" -def _server_major(app: VideoIPathApp) -> Optional[int]: +def _server_major(app: VideoIPathApp) -> int | None: parsed = _parse_version(app._videoipath_connector.videoipath_version) return parsed[0] if parsed is not None else None @@ -74,7 +75,7 @@ def _gate_topology_and_inspect_e2e(request: pytest.FixtureRequest, app: VideoIPa return is_topology = path.name == "test_topology.py" - is_inspect = path.name == "test_inspect.py" or "workflows" in path.parts + is_inspect = path.name == "test_inspect.py" or bool({"workflows", "provisioning"} & set(path.parts)) version = app._videoipath_connector.videoipath_version if is_topology and major > _TOPOLOGY_MAX_MAJOR: diff --git a/tests/e2e/helpers.py b/tests/e2e/helpers.py index 48e2c6a..e3e2074 100644 --- a/tests/e2e/helpers.py +++ b/tests/e2e/helpers.py @@ -285,6 +285,7 @@ def sweep_e2e_namespace(app: "VideoIPathApp") -> None: """ delete_test_tag(app) delete_module_test_tag(app) + _delete_catalog_tags_by_name(app, E2E_TAG) inventory_labels = app.inventory._inventory_api.fetch_devices_user_defined_labels_as_dict() inventory_ids = {i for i, label in inventory_labels.items() if (label or "").startswith(E2E_PREFIX)} @@ -442,6 +443,14 @@ def test_tag_id(app: "VideoIPathApp") -> str: return f"{_e2e_format_tree_id(app)}~~{TEST_TAG_NAME}" +def create_e2e_tag(app: VideoIPathApp) -> str: + """Resolve/create the vipat-e2e catalog leaf; recent servers require full catalog IDs.""" + tree_id = _e2e_format_tree_id(app) + if not catalog_tag_exists(app, path=(tree_id,), name=E2E_TAG): + create_catalog_tag(app, path=(tree_id,), name=E2E_TAG) + return f"{tree_id}~~{E2E_TAG}" + + def module_test_tag_id(app: "VideoIPathApp") -> str: """Full catalog id for the module E2E test tag under the resolved format tree.""" return f"{_e2e_format_tree_id(app)}~~{MODULE_TEST_TAG_NAME}" diff --git a/tests/e2e/provisioning/__init__.py b/tests/e2e/provisioning/__init__.py new file mode 100644 index 0000000..2c883f6 --- /dev/null +++ b/tests/e2e/provisioning/__init__.py @@ -0,0 +1 @@ +"""Live provisioning workflows on server-side mock devices.""" diff --git a/tests/e2e/provisioning/conftest.py b/tests/e2e/provisioning/conftest.py new file mode 100644 index 0000000..994d710 --- /dev/null +++ b/tests/e2e/provisioning/conftest.py @@ -0,0 +1,33 @@ +"""Provisioning scenarios share the standard E2E session gate, allocators, and cleanup.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from ..helpers import create_e2e_tag, create_module_test_tag, module_test_tag_id +from .helpers import LiveProvisioning + +if TYPE_CHECKING: + from collections.abc import Iterator + + from videoipath_automation_tool.apps.videoipath_app import VideoIPathApp + + +@pytest.fixture +def live( + app: VideoIPathApp, e2e_addresses: Iterator[str], e2e_map_origins: Iterator[tuple[int, int]], provisioning_tag: str +) -> LiveProvisioning: + return LiveProvisioning(app, e2e_addresses, next(e2e_map_origins), provisioning_tag) + + +@pytest.fixture(scope="session") +def provisioning_tag(app: VideoIPathApp, e2e_sweep: None) -> str: + return create_e2e_tag(app) + + +@pytest.fixture +def catalog_tag(app: VideoIPathApp) -> str: + create_module_test_tag(app) + return module_test_tag_id(app) diff --git a/tests/e2e/provisioning/fixtures/mock-router.yml b/tests/e2e/provisioning/fixtures/mock-router.yml new file mode 100644 index 0000000..2796841 --- /dev/null +++ b/tests/e2e/provisioning/fixtures/mock-router.yml @@ -0,0 +1,49 @@ +schema_version: 1 +inputs: + ports: + type: integer + default: 2 + modules: + type: integer + default: 1 + x: + type: number + default: 0 + y: + type: number + default: 0 + endpoint_prefix: + type: string + default: endpoint + marker_tag: + type: string +defaults: + inventory: + driver_id: com.nevion.mock-0.1.0 + custom_settings: + num_router_modules: {$input: modules} + num_router_ports: {$input: ports} + num_codec_modules: 0 + topology: + device: + coordinates: + x: {$input: x} + y: {$input: y} + tags: + add: [{$input: marker_tag}] + vertex_processor: + processor_type: example-org.e2e-router + naming: + endpoint_label: "{device.label}-{inputs.endpoint_prefix}-{endpoint.direction}-{endpoint.index:02d}" +variants: + expanded: + inventory: + custom_settings: + num_router_ports: 3 + matrix: + inventory: + custom_settings: + matrix_type: "N:N" + topology: + device: + icon_size: large diff --git a/tests/e2e/provisioning/helpers.py b/tests/e2e/provisioning/helpers.py new file mode 100644 index 0000000..6c6e049 --- /dev/null +++ b/tests/e2e/provisioning/helpers.py @@ -0,0 +1,160 @@ +"""Small live-test fixtures; all writes pass through the real SDK and server.""" + +from __future__ import annotations + +import time +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, TypeVar +from unittest.mock import Mock, patch + +import pytest +from pydantic import BaseModel, ConfigDict + +from videoipath_automation_tool.apps.inspect.errors import InspectEntityNotFoundError +from videoipath_automation_tool.provisioning import ( + Blueprint, + EndpointIdentity, + ProcessorResult, + ProvisioningDevice, + ProvisioningEngine, + TopologyNotReadyError, + VertexEdit, + VertexPatch, + VertexProcessor, +) + +from ..helpers import unique_label + +if TYPE_CHECKING: + from videoipath_automation_tool.apps.inspect.domain.device import InspectDevice + from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice + from videoipath_automation_tool.apps.videoipath_app import VideoIPathApp + from videoipath_automation_tool.provisioning import ApplyResult, ProcessingContext, ProvisioningPlan + +BLUEPRINT_PATH = Path(__file__).parent / "fixtures" / "mock-router.yml" +T = TypeVar("T") + + +class RouterParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class RouterProcessor(VertexProcessor[RouterParams]): + """Expose directional mock router vertices as deterministically named endpoints.""" + + params_model = RouterParams + + def process(self, context: ProcessingContext, params: RouterParams) -> ProcessorResult: + vertices = [v for v in context.find_vertices(kind="router") if v.vertex_type in ("In", "Out")] + if not vertices: + raise TopologyNotReadyError("Mock router directional ports are not discovered yet.") + return ProcessorResult( + vertices=[ + VertexEdit( + vertex_id=vertex.id, + fields=VertexPatch(use_as_endpoint=True, active=True), + endpoint=EndpointIdentity( + direction="RX" if vertex.vertex_type == "In" else "TX", media="video", index=index + ), + ) + for index, vertex in enumerate(vertices, 1) + ] + ) + + +class LiveProvisioning: + """Allocate a small scenario and provide fresh reads, without faking any server behavior.""" + + def __init__(self, app: VideoIPathApp, addresses: Iterator[str], origin: tuple[int, int], tag: str) -> None: + self.app = app + self.tag = tag + self.engine = ProvisioningEngine(app, processors={"example-org.e2e-router": RouterProcessor}) + self.blueprint = Blueprint.load(BLUEPRINT_PATH) + self._addresses = addresses + self._origin = origin + self._slot = 0 + + def device(self, name: str, **fields: Any) -> ProvisioningDevice: + label = unique_label(f"PROV-{name}") + x, y = self._origin + facts = ProvisioningDevice( + key=label, + label=label, + management_address=next(self._addresses), + attributes={"x": x + self._slot * 300, "y": y}, + **fields, + ) + self._slot += 1 + return facts + + def plan(self, device: ProvisioningDevice, **kwargs: Any) -> ProvisioningPlan: + blueprint = kwargs.pop("blueprint", self.blueprint) + inputs = { + "x": device.attributes["x"], + "y": device.attributes["y"], + "marker_tag": self.tag, + **kwargs.pop("inputs", {}), + } + return self.engine.plan(device, blueprint, inputs=inputs, **kwargs) + + def onboard(self, name: str, **kwargs: Any) -> ProvisioningDevice: + device = self.device(name) + result = self.plan(device, **kwargs).apply() + assert_confirmed(result) + assert result.inventory_id is not None + return device.model_copy(update={"inventory_id": result.inventory_id}) + + def inventory(self, device: ProvisioningDevice) -> InventoryDevice: + assert device.inventory_id is not None + return self.app.inventory.get_device(device_id=device.inventory_id, config_only=True) + + def topology(self, device: ProvisioningDevice) -> InspectDevice: + assert device.inventory_id is not None + self.app.inspect.refresh() + found = self.app.inspect.get_device(device.inventory_id) + assert found is not None, f"Provisioned device {device.label} is missing from Inspect" + return found + + +def assert_confirmed(result: ApplyResult) -> None: + assert result.status == "succeeded", result.model_dump() + assert result.verification == "confirmed", result.verification_detail + + +def assert_not_in_topology(app: VideoIPathApp, device_id: str) -> None: + # Collector nodes include Inventory-only devices; an edit form exists only after topology add. + with pytest.raises(InspectEntityNotFoundError): + app.inspect._inspect_api.lookup_inspect_device(device_id) + + +def wait_until(read: Callable[[], T], ready: Callable[[T], bool], description: str, timeout: float = 60) -> T: + """Poll only the expected asynchronous state; API errors propagate immediately.""" + deadline = time.monotonic() + timeout + while True: + value = read() + if ready(value): + return value + if time.monotonic() >= deadline: + raise AssertionError(f"Timed out waiting for {description}; last state: {value!r}") + time.sleep(0.5) + + +@contextmanager +def observe_writes(app: VideoIPathApp) -> Iterator[list[Mock]]: + """Record SDK mutation calls while forwarding every call to its real implementation.""" + targets = [ + (app.inventory, "add_device"), + (app.inventory, "update_device"), + *[ + (app.inspect._inspect_api, name) + for name in ("add_devices", "sync_devices", "update_topology", "assign_tag", "unassign_tag") + ], + ] + with ExitStack() as stack: + yield [stack.enter_context(patch.object(owner, name, wraps=getattr(owner, name))) for owner, name in targets] + + +def assert_no_writes(writes: list[Mock]) -> None: + assert not [call for spy in writes for call in spy.mock_calls] diff --git a/tests/e2e/provisioning/test_connections.py b/tests/e2e/provisioning/test_connections.py new file mode 100644 index 0000000..6771d20 --- /dev/null +++ b/tests/e2e/provisioning/test_connections.py @@ -0,0 +1,157 @@ +"""Live external links, managed edge fields, and deferred peer recovery.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from videoipath_automation_tool.provisioning import ( + Blueprint, + DeviceTarget, + EdgePatch, + PeerEndpoint, + PortSelector, + ProvisioningConnection, +) + +from ..helpers import edges_between +from .helpers import LiveProvisioning, assert_confirmed, assert_no_writes, observe_writes + +if TYPE_CHECKING: + from videoipath_automation_tool.apps.inspect.domain.port import InspectPort + from videoipath_automation_tool.provisioning import ProvisioningDevice + +pytestmark = pytest.mark.e2e + + +def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> None: + local = live.onboard("LINK-A") + peer = live.onboard("LINK-B") + output, input_port = _ports(live, local) + peer_output, peer_input = _ports(live, peer) + document = live.blueprint.model_dump(exclude_unset=True, by_alias=True) + document["defaults"]["topology"]["port_mapping"] = {"send": [{"factory_label": output.factory_label}]} + blueprint = Blueprint.from_dict(document) + local = local.model_copy( + update={ + "connections": [ + ProvisioningConnection( + local="send", + peer=PeerEndpoint( + target=DeviceTarget(device_id=peer.inventory_id), port=PortSelector(port_id=peer_input.id) + ), + direction="outgoing", + fields=EdgePatch(label=f"{local.label}-out", weight=7, capacity=1000, tags=[live.tag]), + ), + ProvisioningConnection( + local=PortSelector(port_id=input_port.id), + peer=PeerEndpoint( + target=DeviceTarget(device_id=peer.inventory_id), port=PortSelector(port_id=peer_output.id) + ), + direction="incoming", + fields=EdgePatch(label=f"{local.label}-in", weight=9, tags=[live.tag]), + ), + ] + } + ) + with observe_writes(live.app) as writes: + plan = live.plan(local, blueprint=blueprint, scope="topology") + assert plan.fully_resolved + assert plan.port_bindings[0].port_id == output.id + assert [c.status for c in plan.connections] == ["planned", "planned"] + assert plan.apply(dry_run=True).status == "planned" + assert_no_writes(writes) + live.app.inspect.refresh() + assert edges_between(live.app, local.inventory_id, peer.inventory_id) == [] + result = plan.apply() + assert_confirmed(result) + assert [c.status for c in result.connections] == ["completed", "completed"] + edge_ids = {edge_id for c in result.connections for edge_id in c.edge_ids} + live.app.inspect.refresh() + edges = edges_between(live.app, local.inventory_id, peer.inventory_id) + assert {e.id for e in edges} == edge_ids + assert {(e.from_port.id, e.to_port.id) for e in edges} == { + (output.id, peer_input.id), + (peer_output.id, input_port.id), + } + outbound_id = result.connections[0].edge_ids[0] + outbound = next(e for e in edges if e.id == outbound_id) + assert (outbound.label, outbound.weight, outbound.capacity) == (f"{local.label}-out", 7, 1000) + assert live.tag in outbound.tags + + with observe_writes(live.app) as writes: + assert live.plan(local, blueprint=blueprint, scope="topology").apply().status == "no_change" + assert_no_writes(writes) + + # An unmanaged field and a second, unrelated link must survive an edge update. + with live.app.inspect.transaction() as tx: + tx.update_edge(outbound_id, description="E2E preserve-edge-description") + tx.commit() + unchanged_id = result.connections[1].edge_ids[0] + live.app.inspect.refresh() + unchanged = next(e for e in live.app.inspect.edges if e.id == unchanged_id) + unchanged_before = (unchanged.label, unchanged.weight, unchanged.tags) + connection = local.connections[0].model_copy(update={"fields": EdgePatch(weight=0, active=False)}) + updated = local.model_copy(update={"connections": [connection]}) + assert_confirmed(live.plan(updated, blueprint=blueprint, scope="topology").apply()) + live.app.inspect.refresh() + edges = {e.id: e for e in edges_between(live.app, local.inventory_id, peer.inventory_id)} + assert set(edges) == edge_ids + assert (edges[outbound_id].weight, edges[outbound_id].active) == (0, False) + assert edges[outbound_id].label == f"{local.label}-out" + assert edges[outbound_id].description == "E2E preserve-edge-description" + assert (edges[unchanged_id].label, edges[unchanged_id].weight, edges[unchanged_id].tags) == unchanged_before + + +def test_missing_peer_requires_a_new_plan(live: LiveProvisioning) -> None: + local = live.onboard("DEFERRED-A") + peer = live.onboard("DEFERRED-B", scope="inventory") + output, _ = _ports(live, local) + local = local.model_copy( + update={ + "connections": [ + ProvisioningConnection( + local=PortSelector(port_id=output.id), + peer=PeerEndpoint( + target=DeviceTarget(device_id=peer.inventory_id), + port=PortSelector(factory_label="Router In 11.1"), + ), + direction="outgoing", + fields=EdgePatch(label=f"{local.label}-deferred", tags=[live.tag]), + ) + ] + } + ) + plan = live.plan(local, scope="topology") + assert not plan.fully_resolved and plan.connections[0].status == "deferred" + with observe_writes(live.app) as writes: + preview = plan.apply(dry_run=True) + assert preview.status == "planned" and preview.replan_required + assert_no_writes(writes) + + # Availability after planning must not silently change the reviewed connection plan. + assert_confirmed(live.plan(peer, scope="topology").apply()) + with observe_writes(live.app) as writes: + pending = plan.apply() + assert pending.status == "partial" and pending.replan_required + assert pending.connections[0].status == "deferred" + assert_no_writes(writes) + live.app.inspect.refresh() + assert edges_between(live.app, local.inventory_id, peer.inventory_id) == [] + resolved = live.plan(local, scope="topology").apply() + assert_confirmed(resolved) + assert not resolved.replan_required + live.app.inspect.refresh() + assert {e.id for e in edges_between(live.app, local.inventory_id, peer.inventory_id)} == set( + resolved.connections[0].edge_ids + ) + assert len(resolved.connections[0].edge_ids) == 1 + + +def _ports(live: LiveProvisioning, device: ProvisioningDevice) -> tuple[InspectPort, InspectPort]: + ports = live.topology(device).ports + return ( + next(p for p in ports if p.factory_label == "Router Out 11.1"), + next(p for p in ports if p.factory_label == "Router In 11.1"), + ) diff --git a/tests/e2e/provisioning/test_lifecycle.py b/tests/e2e/provisioning/test_lifecycle.py new file mode 100644 index 0000000..76d371e --- /dev/null +++ b/tests/e2e/provisioning/test_lifecycle.py @@ -0,0 +1,177 @@ +"""Onboarding, managed updates, naming, and module-scoped configuration.""" + +from __future__ import annotations + +import pytest + +from videoipath_automation_tool.provisioning import ( + ApplyOptions, + Blueprint, + InventorySettings, + ModuleTarget, + NamingScheme, + ProvisioningValidationError, +) + +from .helpers import ( + BLUEPRINT_PATH, + LiveProvisioning, + assert_confirmed, + assert_no_writes, + assert_not_in_topology, + observe_writes, +) + +pytestmark = pytest.mark.e2e + + +def test_preview_apply_and_idempotent_reapply(live: LiveProvisioning) -> None: + device = live.device("LIFECYCLE", description="E2E provisioned router") + naming = NamingScheme(inventory_description="{device.description}", device_description="{device.description}") + with observe_writes(live.app) as writes: + plan = live.plan(device, blueprint=BLUEPRINT_PATH, naming=naming) + assert not plan.fully_resolved + assert plan.phase("inventory").operations[0].action == "create" + assert plan.phase("topology").status == "deferred" + preview = plan.apply(dry_run=True) + assert preview.status == "planned" + assert preview.inventory_id is None + assert_no_writes(writes) + assert live.app.inventory.find_device_id_by_label(device.label, label_search_mode="user_defined_label_only") is None + live.app.inspect.refresh() + assert not [d for d in live.app.inspect.devices if d.label == device.label] + + result = plan.apply() + assert_confirmed(result) + assert result.inventory_id is not None + assert result.topology_device_id == result.inventory_id + assert result.phase("inventory_readiness").status == "completed" + device = device.model_copy(update={"inventory_id": result.inventory_id}) + inventory = live.inventory(device).configuration + assert inventory.label == device.label + assert inventory.address == device.management_address + assert inventory.description == device.description + assert inventory.custom_settings.num_router_ports == 2 + topology = live.topology(device) + assert topology.label == device.label + assert topology.description == device.description + assert live.tag in topology.tags + assert topology.coordinates == {"x": device.attributes["x"], "y": device.attributes["y"]} + endpoints = [v for m in topology.modules for v in m.vertices if v.use_as_endpoint] + assert len(endpoints) == 4 + assert all(v.active and v.label.startswith(f"{device.label}-endpoint-") for v in endpoints) + assert len({v.label for v in endpoints}) == 4 + + with observe_writes(live.app) as writes: + again = live.plan(device, naming=naming) + assert again.fully_resolved and not again.has_changes + assert again.apply().status == "no_change" + assert_no_writes(writes) + assert live.inventory(device).configuration.device_id == result.inventory_id + + +def test_inventory_and_topology_scopes(live: LiveProvisioning) -> None: + device = live.device("SCOPES") + result = live.plan(device, scope="inventory").apply() + assert_confirmed(result) + assert result.phase("topology").status == "skipped" + device = device.model_copy(update={"inventory_id": result.inventory_id}) + original = live.inventory(device).configuration.model_dump() + live.app.inspect.refresh() + assert_not_in_topology(live.app, device.inventory_id) + + assert_confirmed(live.plan(device, scope="topology").apply()) + assert live.topology(device).label == device.label + assert live.inventory(device).configuration.model_dump() == original + + +def test_managed_updates_preserve_unmanaged_fields(live: LiveProvisioning, catalog_tag: str) -> None: + device = live.onboard("MANAGED") + inventory = live.inventory(device) + inventory.configuration.custom_settings.bulk = False + live.app.inventory.update_device(inventory, config_only=True) + with live.app.inspect.transaction() as tx: + tx.update_device(device.inventory_id, description="E2E preserve-description", tags=[live.tag, catalog_tag]) + tx.commit() + + # A variant configures both scopes; the per-instance Inventory setting wins over the variant. + device = device.model_copy( + update={"inventory_overrides": InventorySettings(custom_settings={"matrix_type": "1:1"})} + ) + plan = live.plan(device, variants=["matrix"], scope="inventory") + changes = {change.field: change for op in plan.phase("inventory").operations for change in op.changes} + assert changes["custom_settings.matrix_type"].source == "instance" + assert_confirmed(plan.apply()) + assert live.inventory(device).configuration.custom_settings.matrix_type == "1:1" + assert live.inventory(device).configuration.custom_settings.bulk is False + + # Avoid synchronization here: this test concerns managed-field preservation, not rediscovery. + result = live.plan(device, variants=["matrix"], scope="topology", options=ApplyOptions(sync="none")).apply() + assert_confirmed(result) + topology = live.topology(device) + assert topology.icon_size == "large" + assert topology.description == "E2E preserve-description" + assert set(topology.tags) == {live.tag, catalog_tag} + assert live.inventory(device).configuration.custom_settings.bulk is False + + +def test_endpoint_rename_and_collision_rejection(live: LiveProvisioning) -> None: + device = live.onboard("NAMING") + original = {v.id: v.label for m in live.topology(device).modules for v in m.vertices if v.use_as_endpoint} + with observe_writes(live.app) as writes: + plan = live.plan(device, scope="topology", inputs={"endpoint_prefix": "renamed"}) + assert plan.apply(dry_run=True).status == "planned" + assert_no_writes(writes) + assert {v.id: v.label for m in live.topology(device).modules for v in m.vertices if v.use_as_endpoint} == original + assert_confirmed(plan.apply()) + renamed = {v.id: v.label for m in live.topology(device).modules for v in m.vertices if v.use_as_endpoint} + assert renamed.keys() == original.keys() + assert all(label.startswith(f"{device.label}-renamed-") for label in renamed.values()) + with observe_writes(live.app) as writes: + with pytest.raises(ProvisioningValidationError, match="[Cc]ollision|[Dd]uplicate"): + live.plan(device, scope="topology", naming=NamingScheme(endpoint_label="{device.label}")) + assert_no_writes(writes) + assert {v.id: v.label for m in live.topology(device).modules for v in m.vertices if v.use_as_endpoint} == renamed + + +def test_module_target_tags_and_sibling_isolation(live: LiveProvisioning, catalog_tag: str) -> None: + device = live.onboard("MODULES", inputs={"modules": 2}) + topology = live.topology(device) + module, sibling = sorted(topology.modules, key=lambda item: item.id) + parent_before = (topology.label, topology.description, topology.tags, topology.coordinates) + sibling_before = {v.id: (v.label, v.active, v.use_as_endpoint) for v in sibling.vertices} + inventory_before = live.inventory(device).configuration.model_dump() + target = device.model_copy(update={"topology": ModuleTarget(device_id=topology.id, module_id=module.id)}) + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "defaults": { + "topology": { + "module": {"tags": {"add": [catalog_tag]}}, + "vertex_processor": {"processor_type": "example-org.e2e-router"}, + "naming": {"endpoint_label": "{device.label}-module-{endpoint.direction}-{endpoint.index:02d}"}, + } + }, + } + ) + with observe_writes(live.app) as writes: + plan = live.engine.plan(target, blueprint, scope="topology") + assert plan.apply(dry_run=True).phase("module_tags").status == "planned" + assert_no_writes(writes) + assert_confirmed(plan.apply()) + topology = live.topology(device) + assert catalog_tag in topology.get_module(module.id).tags + assert all("-module-" in v.label for v in topology.get_module(module.id).vertices if v.use_as_endpoint) + assert { + v.id: (v.label, v.active, v.use_as_endpoint) for v in topology.get_module(sibling.id).vertices + } == sibling_before + assert (topology.label, topology.description, topology.tags, topology.coordinates) == parent_before + assert live.inventory(device).configuration.model_dump() == inventory_before + with observe_writes(live.app) as writes: + assert live.engine.apply(target, blueprint, scope="topology").status == "no_change" + assert_no_writes(writes) + remove = Blueprint.from_dict( + {"schema_version": 1, "defaults": {"topology": {"module": {"tags": {"remove": [catalog_tag]}}}}} + ) + assert_confirmed(live.engine.apply(target, remove, scope="topology")) + assert catalog_tag not in live.topology(device).get_module(module.id).tags diff --git a/tests/e2e/provisioning/test_recovery.py b/tests/e2e/provisioning/test_recovery.py new file mode 100644 index 0000000..0a01cb6 --- /dev/null +++ b/tests/e2e/provisioning/test_recovery.py @@ -0,0 +1,199 @@ +"""Actual rediscovery, stale server state, and failed readiness with recoverable IDs.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from videoipath_automation_tool.provisioning import ( + ApplyOptions, + DeviceTarget, + EdgePatch, + InventoryNotReadyError, + InventorySettings, + NamingScheme, + PeerEndpoint, + PortSelector, + ProvisioningApplyError, + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningConnection, +) + +from .helpers import ( + LiveProvisioning, + assert_confirmed, + assert_no_writes, + assert_not_in_topology, + observe_writes, + wait_until, +) + +if TYPE_CHECKING: + from videoipath_automation_tool.provisioning import ProvisioningDevice + +pytestmark = pytest.mark.e2e + + +def test_add_only_sync_of_new_module(live: LiveProvisioning) -> None: + device = live.onboard("ADD-MODULE") + original_vertices = _vertex_ids(live, device) + updated = live.plan(device, inputs={"modules": 2}).apply() + assert updated.status == "partial" and updated.replan_required + _wait_sync(live, device, "add") + result = live.plan(device, inputs={"modules": 2}, options=ApplyOptions(sync="add_only")).apply() + assert_confirmed(result) + assert any(op.action == "sync" for op in result.phase("topology_sync").operations) + vertices = _vertex_ids(live, device) + assert len(vertices) == 8 and original_vertices < vertices + with observe_writes(live.app) as writes: + assert live.plan(device, inputs={"modules": 2}).apply().status == "no_change" + assert_no_writes(writes) + + +def test_rediscovery_add_only_and_reconcile(live: LiveProvisioning) -> None: + device = live.device("SYNC") + initial = live.plan(device, options=ApplyOptions(sync="add_only")).apply() + assert_confirmed(initial) + assert any(op.action == "add_to_topology" for op in initial.phase("topology_sync").operations) + device = device.model_copy(update={"inventory_id": initial.inventory_id}) + original_vertices = _vertex_ids(live, device) + plan = live.plan(device, variants=["expanded"]) + assert not plan.fully_resolved and plan.phase("topology").status == "deferred" + updated = plan.apply() + assert updated.status == "partial" and updated.replan_required + assert updated.verification == "confirmed" + assert updated.phase("inventory").status == "completed" + assert updated.phase("topology").status == "deferred" + assert live.inventory(device).configuration.custom_settings.num_router_ports == 3 + assert _vertex_ids(live, device) == original_vertices + _wait_sync(live, device, "add") + # Expanding the mock router also changes its switching core, so a full sync is required. + added = live.plan(device, variants=["expanded"], options=ApplyOptions(sync="reconcile")).apply() + assert_confirmed(added) + assert not added.replan_required + assert len(_vertex_ids(live, device)) == 6 + + shrinking = live.plan(device).apply() + assert shrinking.status == "partial" and shrinking.replan_required + _wait_sync(live, device, "remove") + with observe_writes(live.app) as writes: + with pytest.raises(ProvisioningCapabilityError, match="add_only"): + live.plan(device) + assert_no_writes(writes) + assert len(_vertex_ids(live, device)) == 6 + assert_confirmed(live.plan(device, options=ApplyOptions(sync="reconcile")).apply()) + assert _vertex_ids(live, device) == original_vertices + with observe_writes(live.app) as writes: + assert live.plan(device).apply().status == "no_change" + assert_no_writes(writes) + + +@pytest.mark.parametrize("scope", ["inventory", "topology"]) +@pytest.mark.parametrize("dry_run", [False, True], ids=["apply", "dry-run"]) +def test_stale_device_plan_preserves_external_change(live: LiveProvisioning, scope: str, dry_run: bool) -> None: + device = live.onboard("STALE") + plan = live.plan( + device, + scope=scope, + naming=NamingScheme( + inventory_description="E2E planned-description", device_description="E2E planned-description" + ), + ) + if scope == "inventory": + record = live.inventory(device) + record.configuration.description = "E2E external-description" + live.app.inventory.update_device(record, config_only=True) + else: + with live.app.inspect.transaction() as tx: + tx.update_device(device.inventory_id, description="E2E external-description") + tx.commit() + with observe_writes(live.app) as writes: + with pytest.raises(ProvisioningApplyError) as caught: + plan.apply(dry_run=dry_run) + assert isinstance(caught.value.__cause__, ProvisioningConflictError) + assert caught.value.result.status == "failed" + assert caught.value.result.phase(scope).status == "failed" + assert_no_writes(writes) + description = ( + live.inventory(device).configuration.description if scope == "inventory" else live.topology(device).description + ) + assert description == "E2E external-description" + + +def test_stale_connection_plan_preserves_external_change(live: LiveProvisioning) -> None: + local, peer = live.onboard("STALE-LINK-A"), live.onboard("STALE-LINK-B") + local = local.model_copy( + update={ + "connections": [ + ProvisioningConnection( + local=PortSelector(factory_label="Router Out 11.1"), + peer=PeerEndpoint( + target=DeviceTarget(device_id=peer.inventory_id), + port=PortSelector(factory_label="Router In 11.1"), + ), + direction="outgoing", + fields=EdgePatch(weight=3, tags=[live.tag]), + ) + ] + } + ) + result = live.plan(local, scope="topology").apply() + assert_confirmed(result) + edge_id = result.connections[0].edge_ids[0] + desired = local.model_copy( + update={"connections": [local.connections[0].model_copy(update={"fields": EdgePatch(weight=5)})]} + ) + plan = live.plan(desired, scope="topology") + with live.app.inspect.transaction() as tx: + tx.update_edge(edge_id, weight=11) + tx.commit() + with observe_writes(live.app) as writes: + with pytest.raises(ProvisioningApplyError) as caught: + plan.apply() + assert isinstance(caught.value.__cause__, ProvisioningConflictError) + assert caught.value.result.status == "failed" + assert_no_writes(writes) + live.app.inspect.refresh() + assert next(e for e in live.app.inspect.edges if e.id == edge_id).weight == 11 + + +def test_inactive_device_timeout_then_recovery(live: LiveProvisioning) -> None: + device = live.device("READINESS", inventory_overrides=InventorySettings(active=False)) + with pytest.raises(ProvisioningApplyError) as caught: + live.plan(device, options=ApplyOptions(inventory_ready_timeout=2, poll_interval=0.25)).apply() + error = caught.value + assert isinstance(error.__cause__, InventoryNotReadyError) + assert error.result.status == "partial" + assert error.result.inventory_id is not None + assert error.result.phase("inventory").status == "completed" + assert error.result.phase("inventory_readiness").status == "failed" + device = device.model_copy(update={"inventory_id": error.result.inventory_id}) + assert live.inventory(device).configuration.active is False + live.app.inspect.refresh() + assert_not_in_topology(live.app, device.inventory_id) + + device = device.model_copy(update={"inventory_overrides": InventorySettings(active=True)}) + assert_confirmed(live.plan(device, scope="inventory").apply()) + assert_confirmed(live.plan(device).apply()) + assert live.topology(device).id == error.result.inventory_id + assert ( + live.app.inventory.find_device_id_by_label(device.label, label_search_mode="user_defined_label_only") + == error.result.inventory_id + ) + with observe_writes(live.app) as writes: + assert live.plan(device).apply().status == "no_change" + assert_no_writes(writes) + + +def _vertex_ids(live: LiveProvisioning, device: ProvisioningDevice) -> set[str]: + return {v.id for m in live.topology(device).modules for v in m.vertices if v.vertex_type in ("In", "Out")} + + +def _wait_sync(live: LiveProvisioning, device: ProvisioningDevice, change: str) -> None: + wait_until( + lambda: live.app.inspect.get_sync_info([device.inventory_id]).get(device.inventory_id), + lambda info: info is not None and bool(getattr(info, change)), + f"mock driver {change} discovery for {device.label}", + ) diff --git a/tests/inspect/test_api.py b/tests/inspect/test_api.py index 87d971a..6e7fa35 100644 --- a/tests/inspect/test_api.py +++ b/tests/inspect/test_api.py @@ -7,7 +7,11 @@ from types import SimpleNamespace from typing import Any +import pytest +from pydantic import ValidationError + from videoipath_automation_tool.apps.inspect.api import InspectAPI +from videoipath_automation_tool.apps.inspect.errors import InspectEntityNotFoundError from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyData @@ -44,18 +48,33 @@ def test_device_skeleton_uses_projection_and_parses(load: Callable[[str], dict[s def test_edge_skeleton_parses(load: Callable[[str], dict[str, Any]]) -> None: edge_items = load("edge_skeleton.json")["data"]["status"]["collector"]["externalEdgesByDeviceKey"]["_items"] - conn, rest = _connector(get_data=_collector(edge_items=edge_items)) + conn, _ = _connector(get_data=_collector(edge_items=edge_items)) api = InspectAPI(conn) edges = api.get_edge_skeleton() assert len(edges) == len(edge_items) def test_device_detail_returns_none_when_absent() -> None: - conn, rest = _connector(get_data=_collector(node_items=[])) + conn, _ = _connector(get_data=_collector(node_items=[])) api = InspectAPI(conn) assert api.get_device_detail("deviceX") is None +def test_null_device_edit_form_reports_missing_entity() -> None: + conn, rest = _connector() + rest._post_data = None + with pytest.raises(InspectEntityNotFoundError) as caught: + InspectAPI(conn).lookup_inspect_device("device-a") + assert caught.value.entity_id == "device-a" + assert caught.value.kind == "device" + + +def test_malformed_device_edit_form_is_not_reported_as_missing() -> None: + conn, _ = _connector(post_data={"fields": {}}) + with pytest.raises(ValidationError): + InspectAPI(conn).lookup_inspect_device("device-a") + + def test_lookup_edges_hits_correct_endpoint(load: Callable[[str], dict[str, Any]]) -> None: conn, rest = _connector(post_data=load("lookup_inspect_edges_by_ids.json")["data"]) api = InspectAPI(conn) diff --git a/tests/inventory/test_inventory_api.py b/tests/inventory/test_inventory_api.py index e4b1c90..3933bd4 100644 --- a/tests/inventory/test_inventory_api.py +++ b/tests/inventory/test_inventory_api.py @@ -6,9 +6,13 @@ from typing import Any import pytest +from pydantic import ValidationError from videoipath_automation_tool.apps.inventory.app.app import InventoryApp -from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError +from videoipath_automation_tool.apps.inventory.errors import ( + InventoryStatusUnavailableError, + InventoryWriteNotAppliedError, +) from videoipath_automation_tool.apps.inventory.inventory_api import InventoryAPI from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice @@ -48,6 +52,25 @@ def _device(device_id: str, address: Any, alternates: Any = None) -> dict[str, A return {"_id": device_id, "config": {"cinfo": cinfo}} +# --- Single-attempt status reads --- + + +@pytest.mark.parametrize("data", [None, {"status": {"devman": {"devices": {"_items": []}}}}]) +def test_missing_device_status_has_a_typed_retryable_error(data: Any) -> None: + api = _api(data) + with pytest.raises(InventoryStatusUnavailableError, match="device1") as info: + api._fetch_device_status("device1") + assert isinstance(info.value, ValueError) + assert len(api.vip_connector.rest.get_calls) == 1 + + +def test_malformed_status_is_not_reported_as_unavailable() -> None: + api = _api({"status": {"devman": {"devices": {"_items": [{"_id": "device1"}]}}}}) + with pytest.raises(ValidationError): + api._fetch_device_status("device1") + assert len(api.vip_connector.rest.get_calls) == 1 + + # --- Address lookups --- diff --git a/tests/blueprints/__init__.py b/tests/provisioning/__init__.py similarity index 100% rename from tests/blueprints/__init__.py rename to tests/provisioning/__init__.py diff --git a/tests/blueprints/conftest.py b/tests/provisioning/conftest.py similarity index 79% rename from tests/blueprints/conftest.py rename to tests/provisioning/conftest.py index 1bf5a02..6d6c3da 100644 --- a/tests/blueprints/conftest.py +++ b/tests/provisioning/conftest.py @@ -1,4 +1,4 @@ -"""Offline harness for blueprint tests. +"""Offline harness for provisioning tests. - ``FakeInventory`` implements the handful of ``InventoryApp`` methods the engine uses, backed by in-memory :class:`InventoryDevice` records. @@ -18,6 +18,7 @@ from videoipath_automation_tool.apps.inspect.app import InspectApp from videoipath_automation_tool.apps.inspect.model.actions import ( + InspectApiLookupEdgesResponse, InspectApiLookupInspectDeviceResponse, InspectApiLookupSyncInfoResponse, InspectApiLookupVerticesResponse, @@ -25,8 +26,10 @@ from videoipath_automation_tool.apps.inspect.model.collector import InspectApiNodeStatusItem from videoipath_automation_tool.apps.inspect.model.common import InspectApiSimpleActionResponse from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyResponse +from videoipath_automation_tool.apps.inventory.errors import InventoryStatusUnavailableError +from videoipath_automation_tool.apps.inventory.model.device_status import DeviceStatus from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice -from videoipath_automation_tool.blueprints import BlueprintEngine +from videoipath_automation_tool.provisioning import ProvisioningEngine from videoipath_automation_tool.utils.cross_app_utils import normalize_address NMOS = "com.nevion.NMOS_multidevice-0.1.0" @@ -45,6 +48,8 @@ def __init__(self) -> None: self.fail_address_read = False self.fail_next_write: Exception | None = None self._next_id = 100 + self.status_sequence: dict[str, list[bool | None | Exception]] = {} + self.status_reads: list[str] = [] def seed(self, device_id: str, **fields: Any) -> InventoryDevice: device = InventoryDevice.create(fields.pop("driver_id", NMOS)) @@ -58,10 +63,23 @@ def seed(self, device_id: str, **fields: Any) -> InventoryDevice: return device def get_device(self, device_id: str, config_only: bool = False) -> InventoryDevice: + assert config_only, "Provisioning must not use Inventory's nested status retry loop." if device_id not in self.devices: raise ValueError(f"No device with id '{device_id}' found in Inventory.") return self.devices[device_id].model_copy(deep=True) + def refresh_device_status(self, device: InventoryDevice) -> InventoryDevice: + device_id = device.device_id + self.status_reads.append(device_id) + sequence = self.status_sequence.get(device_id, [True]) + state = sequence.pop(0) if len(sequence) > 1 else sequence[0] + if isinstance(state, Exception): + raise state + if state is None: + raise InventoryStatusUnavailableError(f"Status for device '{device_id}' is not available yet.") + device.status = inventory_status(device_id, reachable=state) + return device + def add_device(self, device: InventoryDevice, label_check: bool, address_check: bool, config_only: bool) -> Any: assert not label_check and not address_check and config_only self._maybe_fail() @@ -118,6 +136,28 @@ def _maybe_fail(self) -> None: raise error +def inventory_status(device_id: str, *, reachable: bool = True) -> DeviceStatus: + return DeviceStatus.model_validate( + { + "_id": device_id, + "_vid": "status-a", + "canonicalLabel": "device-a", + "deviceInfo": { + "accessUrlOpt": None, + "hw": {"revision": "1", "serial": "synthetic-1"}, + "label": "device-a", + "product": {"name": "device-a", "swBuildTime": None, "swVersion": None}, + "report": [], + }, + "dynamicFn": {}, + "modules": [], + "reachable": reachable, + "softwareInfo": {}, + "url": "http://device-a.example", + } + ) + + # --- Inspect --- @@ -128,6 +168,7 @@ def __init__(self) -> None: self.nodes: dict[str, dict[str, Any]] = {} self.device_forms: dict[str, dict[str, Any]] = {} self.vertex_forms: dict[str, dict[str, Any]] = {} + self.edge_forms: dict[str, dict[str, Any]] = {} self.sync: dict[str, dict[str, Any]] = {} self.discoverable: dict[str, dict[str, Any]] = {} self.calls: list[str] = [] @@ -136,6 +177,10 @@ def __init__(self) -> None: self.fail_tag: str | None = None self.fail_sync_info: Exception | None = None self.fail_sync = False + self.add_visibility_polls = 0 + self.sync_completion_polls = 0 + self._pending_additions: dict[str, int] = {} + self._pending_syncs: dict[str, int] = {} # Setup @@ -148,6 +193,11 @@ def install(self, layout: dict[str, Any]) -> None: def get_device_detail(self, device_id: str) -> InspectApiNodeStatusItem | None: self.calls.append(f"detail:{device_id}") + if device_id in self._pending_additions: + self._pending_additions[device_id] -= 1 + if self._pending_additions[device_id] <= 0: + self._show_added_device(device_id) + del self._pending_additions[device_id] node = self.nodes.get(device_id) return InspectApiNodeStatusItem.model_validate(copy.deepcopy(node)) if node is not None else None @@ -168,6 +218,12 @@ def lookup_sync_info(self, device_ids: list[str]) -> InspectApiLookupSyncInfoRes if self.fail_sync_info is not None: error, self.fail_sync_info = self.fail_sync_info, None raise error + for device_id in device_ids: + if device_id in self._pending_syncs: + self._pending_syncs[device_id] -= 1 + if self._pending_syncs[device_id] <= 0: + self._finish_sync(device_id) + del self._pending_syncs[device_id] data = {d: {"label": d, **self.sync[d]} for d in device_ids if d in self.sync} return InspectApiLookupSyncInfoResponse.model_validate({"data": data, "header": HEADER}) @@ -177,6 +233,14 @@ def get_device_skeleton(self) -> list[InspectApiNodeStatusItem]: def get_edge_skeleton(self) -> list[Any]: return [] + def get_edge_pair(self, pair_id: str) -> None: + return None + + def lookup_edges(self, edge_ids: list[str]) -> InspectApiLookupEdgesResponse: + self.calls.append(f"lookup_edges:{len(edge_ids)}") + data = {eid: {"edge": copy.deepcopy(self.edge_forms[eid])} for eid in edge_ids if eid in self.edge_forms} + return InspectApiLookupEdgesResponse.model_validate({"data": data, "header": HEADER}) + # Writes def update_topology(self, delta: Any) -> InspectApiUpdateTopologyResponse: @@ -188,6 +252,8 @@ def update_topology(self, delta: Any) -> InspectApiUpdateTopologyResponse: self.device_forms[device_id] = form.model_dump(mode="json") for vertex_id, form in delta.replaceVertices.items(): self.vertex_forms[vertex_id]["fields"] = form.model_dump(mode="json") + for edge_id, form in delta.replaceEdges.items(): + self.edge_forms[edge_id] = form.model_dump(mode="json") result = {"msg": [], "ok": True} data = {"items": [], "res": result, "validation": {"createIds": [], "details": {}, "result": result}} return InspectApiUpdateTopologyResponse.model_validate({"data": data, "header": HEADER}) @@ -204,9 +270,10 @@ def add_devices(self, items: list[Any]) -> InspectApiSimpleActionResponse: pending = self.discoverable.get(item.id) if pending is None: return _action(False, ["device not discovered"]) - self.nodes[item.id] = {"_id": item.id, "modules": {}} - self.device_forms[item.id] = copy.deepcopy(pending["device_form"]) - self.sync[item.id] = {"add": {"modules": 1}, "update": {}, "remove": {}} + if self.add_visibility_polls: + self._pending_additions[item.id] = self.add_visibility_polls + else: + self._show_added_device(item.id) return _action(True) def sync_devices(self, device_ids: list[str], add_only: bool = True, conflict_strategy: int = 0) -> Any: @@ -214,12 +281,23 @@ def sync_devices(self, device_ids: list[str], add_only: bool = True, conflict_st self.writes.append(("sync_devices", (device_id, add_only))) if self.fail_sync: return _action(False, ["sync rejected"]) - pending = self.discoverable.pop(device_id, None) - if pending is not None: - self.install(pending) - self.sync.pop(device_id, None) + if self.sync_completion_polls: + self._pending_syncs[device_id] = self.sync_completion_polls + else: + self._finish_sync(device_id) return _action(True) + def _show_added_device(self, device_id: str) -> None: + self.nodes[device_id] = {"_id": device_id, "modules": {}} + self.device_forms[device_id] = copy.deepcopy(self.discoverable[device_id]["device_form"]) + self.sync[device_id] = {"add": {"modules": 1}, "update": {}, "remove": {}} + + def _finish_sync(self, device_id: str) -> None: + pending = self.discoverable.pop(device_id, None) + if pending is not None: + self.install(pending) + self.sync.pop(device_id, None) + def _tag(self, action: str, tag_id: str, element_ids: list[str]) -> InspectApiSimpleActionResponse: self.writes.append((action, (tag_id, tuple(element_ids)))) if self.fail_tag == tag_id: @@ -370,8 +448,8 @@ def app(inventory: FakeInventory, server: FakeInspectServer) -> SimpleNamespace: @pytest.fixture -def engine(app: SimpleNamespace) -> BlueprintEngine: - engine = BlueprintEngine(app) +def engine(app: SimpleNamespace) -> ProvisioningEngine: + engine = ProvisioningEngine(app) clock = {"now": 0.0} engine._clock = lambda: clock["now"] engine._sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) diff --git a/tests/blueprints/test_blueprint_inputs.py b/tests/provisioning/test_blueprint_inputs.py similarity index 75% rename from tests/blueprints/test_blueprint_inputs.py rename to tests/provisioning/test_blueprint_inputs.py index aba1d15..07ee31d 100644 --- a/tests/blueprints/test_blueprint_inputs.py +++ b/tests/provisioning/test_blueprint_inputs.py @@ -7,15 +7,20 @@ import pytest -from tests.blueprints.conftest import NMOS, FakeInventory -from videoipath_automation_tool.blueprints import Blueprint, BlueprintDevice, BlueprintEngine, BlueprintValidationError +from tests.provisioning.conftest import NMOS, FakeInventory +from videoipath_automation_tool.provisioning import ( + Blueprint, + ProvisioningDevice, + ProvisioningEngine, + ProvisioningValidationError, +) @pytest.fixture def blueprint_path(tmp_path: Path) -> Path: path = tmp_path / "device-blueprint.yml" path.write_text( - f"schema_version: 1\ninventory:\n default:\n driver_id: {NMOS}\n custom_settings: {{port: 8080}}\n", + f"schema_version: 1\ndefaults:\n inventory:\n driver_id: {NMOS}\n custom_settings: {{port: 8080}}\n", encoding="utf-8", ) return path @@ -26,8 +31,8 @@ def test_file_inputs_match_blueprint_instances( blueprint_path: Path, inventory: FakeInventory, source_type: type[str | PurePath] ) -> None: inventory.seed("device1", label="device-a", custom={"port": 80}) - engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) - device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + engine = ProvisioningEngine(SimpleNamespace(inventory=inventory)) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") source = source_type(blueprint_path) engine.validate(source) @@ -43,8 +48,8 @@ def test_file_inputs_match_blueprint_instances( def test_apply_loads_file_once(blueprint_path: Path, inventory: FakeInventory, monkeypatch: pytest.MonkeyPatch) -> None: inventory.seed("device1", label="device-a", custom={"port": 80}) - engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) - device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + engine = ProvisioningEngine(SimpleNamespace(inventory=inventory)) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") original_load = Blueprint.load loaded: list[Path] = [] @@ -61,8 +66,8 @@ def test_plans_keep_loaded_content_when_file_changes_or_disappears( blueprint_path: Path, inventory: FakeInventory ) -> None: inventory.seed("device1", label="device-a", custom={"port": 80}) - engine = BlueprintEngine(SimpleNamespace(inventory=inventory)) - device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + engine = ProvisioningEngine(SimpleNamespace(inventory=inventory)) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") first = engine.plan(device, blueprint_path, scope="inventory") blueprint_path.write_text(blueprint_path.read_text(encoding="utf-8").replace("8080", "9090"), encoding="utf-8") @@ -80,8 +85,8 @@ def test_plans_keep_loaded_content_when_file_changes_or_disappears( def test_file_errors_happen_before_app_access( blueprint_path: Path, method: str, monkeypatch: pytest.MonkeyPatch ) -> None: - engine = BlueprintEngine(SimpleNamespace()) - device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + engine = ProvisioningEngine(SimpleNamespace()) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") operation = getattr(engine, method) args = () if method == "validate" else (device,) monkeypatch.chdir(blueprint_path.parent) @@ -92,15 +97,15 @@ def test_file_errors_happen_before_app_access( operation(*args, "schema_version: 1") # Strings are paths, not implicit YAML documents. blueprint_path.write_text("schema_version: 1\nschema_version: 1\n", encoding="utf-8") - with pytest.raises(BlueprintValidationError, match="duplicate key") as error: + with pytest.raises(ProvisioningValidationError, match="duplicate key") as error: operation(*args, blueprint_path.name) assert blueprint_path.name in str(error.value) @pytest.mark.parametrize("method", ["plan", "apply", "validate"]) def test_invalid_blueprint_input_has_a_useful_type_error(method: str) -> None: - engine = BlueprintEngine(SimpleNamespace()) - device = BlueprintDevice(key="device-a", label="device-a", inventory_id="device1") + engine = ProvisioningEngine(SimpleNamespace()) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") args = () if method == "validate" else (device,) with pytest.raises(TypeError, match="Blueprint instance or a YAML file path"): getattr(engine, method)(*args, {"schema_version": 1}) diff --git a/tests/provisioning/test_connection_inputs.py b/tests/provisioning/test_connection_inputs.py new file mode 100644 index 0000000..d5c837d --- /dev/null +++ b/tests/provisioning/test_connection_inputs.py @@ -0,0 +1,159 @@ +"""Schema-v2 binding, overlay and source-location behavior of generic port mappings.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests.provisioning.conftest import FakeInspectServer, FakeInventory +from tests.provisioning.test_connections import _connection, _device, _seed +from videoipath_automation_tool.provisioning import Blueprint, ProvisioningEngine, ProvisioningValidationError +from videoipath_automation_tool.provisioning.resolution import resolve_blueprint + + +@pytest.mark.parametrize("position", ["value", "selector", "list", "mapping"]) +def test_typed_mapping_input_positions( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + position: str, +) -> None: + _seed(inventory, server) + expression = {"$input": "ports"} + mappings = { + "value": {"uplink": [{"factory_label": expression}]}, + "selector": {"uplink": [expression]}, + "list": {"uplink": expression}, + "mapping": expression, + } + values = { + "value": "P1", + "selector": {"factory_label": "P1"}, + "list": [{"factory_label": "P1"}], + "mapping": {"uplink": [{"factory_label": "P1"}]}, + } + definitions = { + "value": {"type": "string"}, + "selector": {"type": "object", "additionalProperties": True}, + "list": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, + "mapping": {"type": "object", "additionalProperties": True}, + } + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "inputs": {"ports": definitions[position]}, + "defaults": {"topology": {"port_mapping": mappings[position]}}, + } + ) + inputs = {"ports": values[position]} + resolved = resolve_blueprint(blueprint, inputs=inputs) + assert any("input:ports" in source for source in resolved.topology.provenance.values()) + plan = engine.plan(_device(_connection()), blueprint, inputs=inputs) + inputs["ports"] = "changed" + result = plan.apply() + assert result.connections[0].local.port_id == "device1.dev.0.P1" + assert result.ok and len(server.edge_forms) == 2 + + +def test_mapping_overlay_order_lists_replace_and_atomic_inputs() -> None: + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "inputs": { + "ports": { + "type": "object", + "additionalProperties": True, + "default": {"replacement": [{"factory_label": "P3"}]}, + } + }, + "defaults": { + "topology": { + "port_mapping": { + "uplink": [{"factory_label": "missing"}, {"factory_label": "P1"}], + "other": [{"factory_label": "P2"}], + } + } + }, + "variants": { + "first": {"topology": {"port_mapping": {"uplink": [{"factory_label": "P2"}]}}}, + "second": {"topology": {"port_mapping": {"uplink": [{"factory_label": "P1"}]}}}, + "input": {"topology": {"port_mapping": {"$input": "ports"}}}, + }, + } + ) + resolved = resolve_blueprint(blueprint, variants=["first", "second"]) + assert [item.factory_label for item in resolved.topology.config.port_mapping["uplink"]] == ["P1"] + assert set(resolved.topology.config.port_mapping) == {"uplink", "other"} + assert resolved.topology.provenance["port_mapping.uplink"] == "variant:second" + assert set(resolve_blueprint(blueprint, variants=["input"]).topology.config.port_mapping) == {"replacement"} + assert len(blueprint.defaults.topology.port_mapping["uplink"]) == 2 + + +def test_literal_and_inserted_operator_text_is_not_reinterpreted() -> None: + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "inputs": {"label": {"type": "string", "default": "$input: not-an-operator"}}, + "defaults": { + "topology": { + "port_mapping": { + "literal": {"$literal": [{"factory_label": "$input: literal"}]}, + "inserted": [{"factory_label": {"$input": "label"}}], + } + } + }, + } + ) + mapping = resolve_blueprint(blueprint).topology.config.port_mapping + assert mapping["literal"][0].factory_label == "$input: literal" + assert mapping["inserted"][0].factory_label == "$input: not-an-operator" + + +def test_runtime_selector_errors_retain_yaml_origin_and_fail_before_io() -> None: + blueprint = Blueprint.from_yaml( + """schema_version: 1 +inputs: + selector: + type: object + additionalProperties: true +defaults: + topology: + port_mapping: + uplink: + - {$input: selector} +""", + source="device.yml", + ) + + class NoIO: + def __getattr__(self, key: str) -> Any: + raise AssertionError(f"unexpected I/O: {key}") + + with pytest.raises(ProvisioningValidationError, match="exactly one") as error: + ProvisioningEngine(NoIO()).plan( + _device(_connection()), blueprint, inputs={"selector": {"port_id": "port-a", "factory_label": "P1"}} + ) + issue = error.value.issues[0] + assert issue.source == "device.yml" and issue.line == 10 + assert issue.path.startswith("defaults.topology.port_mapping") + + +def test_missing_mapping_input_is_only_required_in_effective_scope() -> None: + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "inputs": {"port": {"type": "string"}}, + "defaults": { + "inventory": {"driver_id": "com.nevion.NMOS_multidevice-0.1.0"}, + "topology": {"port_mapping": {"uplink": [{"factory_label": {"$input": "port"}}]}}, + }, + "variants": {"fixed": {"topology": {"port_mapping": {"uplink": [{"factory_label": "P1"}]}}}}, + } + ) + assert resolve_blueprint(blueprint, scope="inventory").topology is None + with pytest.raises(ProvisioningValidationError, match="Required input is missing"): + resolve_blueprint(blueprint) + assert ( + resolve_blueprint(blueprint, variants=["fixed"]).topology.config.port_mapping["uplink"][0].factory_label == "P1" + ) diff --git a/tests/provisioning/test_connections.py b/tests/provisioning/test_connections.py new file mode 100644 index 0000000..0c10406 --- /dev/null +++ b/tests/provisioning/test_connections.py @@ -0,0 +1,450 @@ +"""Generic connections through the real provisioning engine and Inspect transaction, offline.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from pydantic import ValidationError + +from tests.provisioning.conftest import HEADER, FakeInspectServer, FakeInventory, build_layout, matrox_layout +from videoipath_automation_tool.apps.inspect.errors import InspectCommitError +from videoipath_automation_tool.apps.inspect.model.actions import InspectApiEdgeForm +from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyResponse +from videoipath_automation_tool.provisioning import ( + Blueprint, + DeviceTarget, + EdgePatch, + ModuleTarget, + PeerEndpoint, + PortSelector, + ProvisioningApplyError, + ProvisioningConnection, + ProvisioningDevice, + ProvisioningEngine, + ProvisioningTargetError, + ProvisioningValidationError, + TagDelta, +) + + +def _blueprint(**topology: Any) -> Blueprint: + config = {"port_mapping": {"uplink": [{"factory_label": "P1"}]}, **topology} + return Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": config}}) + + +def _connection(**kwargs: Any) -> ProvisioningConnection: + values = { + "local": "uplink", + "peer": PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="P1")), + **kwargs, + } + return ProvisioningConnection(**values) + + +def _device(*connections: ProvisioningConnection, **kwargs: Any) -> ProvisioningDevice: + return ProvisioningDevice( + key="device-a", label="device-a", inventory_id="device1", connections=list(connections), **kwargs + ) + + +def _seed(inventory: FakeInventory, server: FakeInspectServer, *, peer: bool = True) -> None: + inventory.seed("device1", label="device-a") + server.install(matrox_layout("device1")) + if peer: + server.install(matrox_layout("device2")) + + +def test_bidirectional_create_shared_commit_and_idempotence( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + device = _device(_connection(fields=EdgePatch(weight=0, capacity=0, bandwidth=0.0, active=False))) + blueprint = _blueprint(vertices=[{"factory_label": "Video Sender", "fields": {"active": False}}]) + plan = engine.plan(device, blueprint) + assert not server.writes + assert plan.fully_resolved and len(plan.connections[0].edge_ids) == 2 + assert plan.interface_bindings == [] and plan.port_bindings[0].key == "uplink" + assert "connection 0 [planned]" in plan.summary() + assert plan.apply(dry_run=True).connections[0].status == "planned" + assert not server.writes + result = plan.apply() + assert result.status == "succeeded" and result.verification == "confirmed" + assert result.connections[0].status == "completed" + assert len(server.writes) == 1 + delta = server.writes[0][1] + assert delta["replaceDevices"] and delta["replaceVertices"] and len(delta["replaceEdges"]) == 2 + assert all( + not form["active"] and form["capacity"] == 0 and form["weight"] == 0 for form in server.edge_forms.values() + ) + assert engine.apply(device, blueprint).status == "no_change" + assert len(server.writes) == 1 + + +@pytest.mark.parametrize("media", ["video", "audio"]) +@pytest.mark.parametrize("direction", ["auto", "outgoing"]) +def test_directed_media_without_direction_suffix( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + media: str, + direction: str, +) -> None: + inventory.seed("device1") + for device_id, side in [("device1", "Out"), ("device2", "In")]: + server.install( + build_layout(device_id, {f"{device_id}.dev.0": [("port-1", "media-1", [("vertex", side, "codec", media)])]}) + ) + peer = PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="media-1")) + result = engine.apply( + _device(_connection(local=PortSelector(factory_label="media-1"), peer=peer, direction=direction)), + _blueprint(port_mapping={}), + ) + assert result.connections[0].edge_ids == ["device1.0.port-1.vertex::device2.0.port-1.vertex"] + assert len(server.edge_forms) == 1 + + +@pytest.mark.parametrize("direction,count", [("incoming", 1), ("outgoing", 1), ("bidirectional", 2)]) +def test_explicit_directions( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, direction: str, count: int +) -> None: + _seed(inventory, server) + result = engine.apply(_device(_connection(direction=direction)), _blueprint()) + assert len(result.connections[0].edge_ids) == count + if direction == "incoming": + assert result.connections[0].edge_ids == ["device2.0.P1.out::device1.0.P1.in"] + + +def test_managed_updates_preserve_other_edges_and_nested_fields( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + engine.apply(_device(_connection()), _blueprint()) + for form in server.edge_forms.values(): + form["descriptor"]["desc"] = "keep-description" + form["fDescriptor"]["label"] = "factory-edge" + form["tags"] = ["tag-a", "tag-b"] + form["weightFactors"] = {"bandwidth": {"weight": 5}, "service": {"max": 43, "weight": 3}} + foreign = "device1.0.P2.out::device2.0.P2.in" + server.edge_forms[foreign] = InspectApiEdgeForm(fromId="device1.0.P2.out", toId="device2.0.P2.in").model_dump() + fields = EdgePatch( + label="edge-a", + tags=TagDelta(add=["tag-c"], remove=["tag-a"]), + weight_per_service=0, + conflict_priority="high", + include_formats=[], + exclude_formats=["format-a"], + redundancy_mode="Any", + bandwidth_weight_factor=0, + ) + result = engine.apply(_device(_connection(fields=fields)), _blueprint()) + for edge_id in result.connections[0].edge_ids: + form = server.edge_forms[edge_id] + assert form["descriptor"] == {"label": "edge-a", "desc": "keep-description"} + assert form["fDescriptor"]["label"] == "factory-edge" + assert form["tags"] == ["tag-b", "tag-c"] + assert form["weightFactors"] == {"bandwidth": {"weight": 0}, "service": {"max": 43, "weight": 0}} + assert form["conflictPri"] == 1 and form["excludeFormats"] == ["format-a"] + assert foreign in server.edge_forms + assert engine.apply(_device(_connection(fields=fields)), _blueprint()).status == "no_change" + assert engine.apply(_device(), _blueprint()).status == "no_change" + assert len(server.edge_forms) == 3 + + +def test_duplicate_connections_coalesce_or_reject_conflicts( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + connection = _connection(fields=EdgePatch(weight=3)) + plan = engine.plan(_device(connection, connection, _connection(fields=EdgePatch(label="edge-a"))), _blueprint()) + assert len([op for op in plan.phase("topology").operations if op.entity_kind == "edge"]) == 2 + with pytest.raises(ProvisioningValidationError, match="Conflicting 'weight'"): + engine.plan(_device(connection, _connection(fields=EdgePatch(weight=4))), _blueprint()) + assert not server.writes + + +def test_missing_peer_is_partial_and_frozen_until_replan( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server, peer=False) + device, blueprint = _device(_connection()), _blueprint() + plan = engine.plan(device, blueprint) + assert not plan.fully_resolved and plan.connections[0].status == "deferred" + preview = plan.apply(dry_run=True) + assert preview.status == "planned" and preview.replan_required and not server.writes + server.install(matrox_layout("device2")) + result = plan.apply() + assert result.status == "partial" and result.replan_required and not result.ok + assert not server.edge_forms + assert result.connections[0].local and result.connections[0].peer is None + assert engine.apply(device, blueprint).status == "succeeded" + + +def test_pending_without_any_write_is_still_partial( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server, peer=False) + engine.apply(_device(), _blueprint()) + server.writes.clear() + result = engine.apply(_device(_connection()), _blueprint()) + assert result.status == "partial" and result.phase("topology").status == "deferred" + assert result.verification == "not_applicable" and not server.writes + + +def test_mixed_ready_and_pending_connections( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + missing = PeerEndpoint(target=DeviceTarget(device_id="device3"), port=PortSelector(factory_label="P1")) + result = engine.apply(_device(_connection(), _connection(peer=missing)), _blueprint()) + assert result.status == "partial" and result.verification == "confirmed" + assert [state.status for state in result.connections] == ["completed", "deferred"] + assert len(server.edge_forms) == 2 + + +@pytest.mark.parametrize("missing", ["module", "ports"]) +def test_undiscovered_peer_module_or_ports_is_pending( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + missing: str, +) -> None: + _seed(inventory, server) + if missing == "ports": + server.nodes["device2"]["modules"]["device2.dev.0"]["ports"] = {} + target = ModuleTarget(device_id="device2", module_id="device2.dev.1" if missing == "module" else "device2.dev.0") + peer = PeerEndpoint(target=target, port=PortSelector(factory_label="P1")) + assert engine.apply(_device(_connection(peer=peer)), _blueprint()).connections[0].status == "deferred" + + +def test_peer_selector_miss_and_transport_error_are_not_pending( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _seed(inventory, server) + peer = PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="absent")) + with pytest.raises(ProvisioningTargetError, match="no match"): + engine.plan(_device(_connection(peer=peer)), _blueprint()) + + def fail(device_id: str) -> None: + raise OSError("read failed") + + monkeypatch.setattr(server, "get_device_detail", fail) + with pytest.raises(OSError, match="read failed"): + engine.plan(_device(_connection()), _blueprint()) + assert not server.writes + + +@pytest.mark.parametrize("change", ["peer", "edge_created", "edge_updated", "edge_removed"]) +@pytest.mark.parametrize("dry_run", [False, True]) +def test_stale_peer_and_edge_baselines( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + change: str, + dry_run: bool, +) -> None: + _seed(inventory, server) + device, blueprint = _device(_connection()), _blueprint() + if change in ("edge_updated", "edge_removed"): + engine.apply(device, blueprint) + plan = engine.plan(device, blueprint) + if change == "peer": + server.vertex_forms["device2.0.P1.out"]["fields"]["active"] = False + elif change == "edge_created": + server.edge_forms["device1.0.P1.out::device2.0.P1.in"] = InspectApiEdgeForm( + fromId="device1.0.P1.out", toId="device2.0.P1.in" + ).model_dump() + elif change == "edge_updated": + next(iter(server.edge_forms.values()))["weight"] = 8 + else: + server.edge_forms.pop(next(iter(server.edge_forms))) + server.writes.clear() + with pytest.raises(ProvisioningApplyError, match="changed"): + plan.apply(dry_run=dry_run) + assert not server.writes + + +@pytest.mark.parametrize("kind", ["edge", "vertex", "module", "device"]) +def test_pending_snapshot_edits_on_connection_entities( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + app: SimpleNamespace, + kind: str, +) -> None: + _seed(inventory, server) + plan = engine.plan(_device(_connection()), _blueprint()) + entity_id = { + "edge": plan.connections[0].edge_ids[0], + "vertex": "device2.0.P1.in", + "module": "device2.dev.0", + "device": "device2", + }[kind] + app.inspect._get_snapshot().stage_edit(kind, entity_id, "label", "pending") + with pytest.raises(ProvisioningApplyError, match="Pending uncommitted"): + plan.apply() + assert not server.writes + + +def test_plan_copies_connections_and_edge_fields( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + fields = EdgePatch(tags=["tag-a"]) + device = _device(_connection(fields=fields)) + plan = engine.plan(device, _blueprint()) + fields.tags.append("tag-b") + device.connections.clear() + plan.connections[0].connection.fields.tags.append("tag-c") + result = plan.apply() + assert all(form["tags"] == ["tag-a"] for form in server.edge_forms.values()) + assert result.connections[0].status == "completed" + + +def test_local_discovery_materializes_edges( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + inventory.seed("device1") + server.discoverable["device1"] = matrox_layout("device1") + server.install(matrox_layout("device2")) + plan = engine.plan(_device(_connection()), _blueprint()) + assert not plan.fully_resolved + assert plan.apply(dry_run=True).connections[0].status == "deferred" + assert not server.writes + result = plan.apply() + assert result.materialized and result.status == "succeeded" and len(server.edge_forms) == 2 + + +def test_failure_and_verification_status( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _seed(inventory, server) + rejection = InspectApiUpdateTopologyResponse.model_validate( + { + "header": HEADER, + "data": { + "items": [], + "res": {"ok": False, "msg": ["rejected"]}, + "validation": {"result": {"ok": True, "msg": []}, "createIds": [], "details": {}}, + }, + } + ) + server.fail_commit = InspectCommitError(rejection) + with pytest.raises(ProvisioningApplyError) as error: + engine.apply(_device(_connection()), _blueprint()) + assert error.value.result.connections[0].status == "failed" + assert not server.edge_forms + server.fail_commit = None + original = server.update_topology + + def discard_edges(delta: Any) -> Any: + result = original(delta) + server.edge_forms.clear() + return result + + monkeypatch.setattr(server, "update_topology", discard_edges) + result = engine.apply(_device(_connection()), _blueprint()) + assert result.status == "succeeded" and result.verification == "unconfirmed" + assert "fromId" in result.verification_detail + + +def test_legacy_ip_mapping_can_supply_connection_without_changing_processor_inputs( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + blueprint = _blueprint( + port_mapping={"media": [{"factory_label": "Video Sender"}]}, + ip_vertex_mapping={"uplink": ["P1"], "backup": ["P2"]}, + vertex_processor={"processor_type": "matrox.convertip.default", "params": {"redundant_streams": True}}, + ) + plan = engine.plan(_device(_connection()), blueprint) + assert [binding.key for binding in plan.interface_bindings] == ["uplink", "backup"] + assert [binding.key for binding in plan.port_bindings] == ["media"] + assert plan.apply().ok + + +@pytest.mark.parametrize("fields", [{}, {"port_id": "port-a", "factory_label": "port-a"}]) +def test_port_selector_validation(fields: dict[str, Any]) -> None: + with pytest.raises(ValidationError, match="exactly one"): + PortSelector(**fields) + + +def test_local_fallback_module_scope_and_ambiguity( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + second = matrox_layout("device1", module="device1.dev.1") + server.nodes["device1"]["modules"].update(second["node"]["modules"]) + server.vertex_forms.update(second["vertex_forms"]) + with pytest.raises(ProvisioningTargetError, match="multiple ports"): + engine.plan(_device(_connection()), _blueprint()) + blueprint = _blueprint( + port_mapping={"uplink": [{"factory_label": "missing"}, {"factory_label": "P{module.position}", "kind": "ip"}]} + ) + device = _device( + _connection(), module_position="1", topology=ModuleTarget(device_id="device1", module_id="device1.dev.0") + ) + assert engine.apply(device, blueprint).connections[0].local.port_id == "device1.dev.0.P1" + wrong = _connection(local=PortSelector(port_id="device1.dev.1.P1")) + with pytest.raises(ProvisioningTargetError, match="no match"): + engine.plan(_device(wrong, topology=device.topology), _blueprint(port_mapping={})) + + +def test_invalid_input_scope_and_mapping_keys_fail_before_io() -> None: + class NoIO: + def __getattr__(self, key: str) -> Any: + raise AssertionError(f"unexpected I/O: {key}") + + engine = ProvisioningEngine(NoIO()) + with pytest.raises(ProvisioningValidationError, match="Unknown local port"): + engine.plan(_device(_connection(local="unknown")), _blueprint()) + with pytest.raises(ProvisioningValidationError, match="require a topology section"): + engine.plan( + _device(_connection()), + Blueprint.from_dict( + {"schema_version": 1, "defaults": {"inventory": {"driver_id": "com.nevion.NMOS_multidevice-0.1.0"}}} + ), + ) + with pytest.raises(ProvisioningValidationError, match="distinct keys"): + _blueprint(ip_vertex_mapping={"uplink": ["P1"]}) + + +def test_inventory_scope_skips_connections( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + blueprint = Blueprint.from_dict( + {"schema_version": 1, "defaults": {"inventory": {"driver_id": "com.nevion.NMOS_multidevice-0.1.0"}}} + ) + plan = engine.plan(_device(_connection()), blueprint, scope="inventory") + assert not server.calls and not plan.connections and plan.fully_resolved + assert plan.skipped_sections["topology"] == "not in scope 'inventory'" diff --git a/tests/blueprints/test_engine.py b/tests/provisioning/test_engine.py similarity index 69% rename from tests/blueprints/test_engine.py rename to tests/provisioning/test_engine.py index 8afcf39..73a4d7d 100644 --- a/tests/blueprints/test_engine.py +++ b/tests/provisioning/test_engine.py @@ -11,22 +11,13 @@ import pytest from pydantic import BaseModel, ConfigDict -from tests.blueprints.conftest import NMOS, FakeInspectServer, FakeInventory, make_inspect_app, matrox_layout +from tests.provisioning.conftest import NMOS, FakeInspectServer, FakeInventory, make_inspect_app, matrox_layout from videoipath_automation_tool.apps.inspect.snapshot import InspectSnapshot from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( AlternativeAddress, ApplyOptions, Blueprint, - BlueprintApplyError, - BlueprintCapabilityError, - BlueprintConflictError, - BlueprintDevice, - BlueprintEngine, - BlueprintError, - BlueprintPlan, - BlueprintTargetError, - BlueprintValidationError, Credentials, DeviceRecord, DeviceTarget, @@ -36,19 +27,28 @@ ProcessingContext, ProcessorInputError, ProcessorResult, + ProvisioningApplyError, + ProvisioningCapabilityError, + ProvisioningConflictError, + ProvisioningDevice, + ProvisioningEngine, + ProvisioningError, + ProvisioningPlan, + ProvisioningTargetError, + ProvisioningValidationError, TopologyNotReadyError, VertexEdit, VertexPatch, VertexProcessor, published_json_schema, ) -from videoipath_automation_tool.blueprints.inspect import ScopeData, resolve_interfaces +from videoipath_automation_tool.provisioning.inspect import ScopeData, resolve_interfaces MATROX = { "schema_version": 1, - "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": 8080}}}, - "topology": { - "default": { + "defaults": { + "inventory": {"driver_id": NMOS, "custom_settings": {"port": 8080}}, + "topology": { "device": {"icon_size": "medium"}, "ip_vertex_mapping": {"stream-a": ["P1"], "stream-b": ["P2"]}, "vertex_processor": { @@ -56,21 +56,21 @@ "params": {"redundant_streams": True, "video_sender_tags": ["video-tag-b"]}, }, }, - "receiver": {"vertex_processor": {"params": {"mode": "rx"}}}, }, + "variants": {"receiver": {"topology": {"vertex_processor": {"params": {"mode": "rx"}}}}}, } def _blueprint(document: dict[str, Any] = MATROX, **topology_default: Any) -> Blueprint: data = { **document, - "topology": {**document["topology"], "default": {**document["topology"]["default"], **topology_default}}, + "defaults": {**document["defaults"], "topology": {**document["defaults"]["topology"], **topology_default}}, } return Blueprint.from_dict(data) -def _device(**fields: Any) -> BlueprintDevice: - return BlueprintDevice(**{"key": "key-a", "label": "device-a", "inventory_id": "device1", **fields}) +def _device(**fields: Any) -> ProvisioningDevice: + return ProvisioningDevice(**{"key": "key-a", "label": "device-a", "inventory_id": "device1", **fields}) def _existing(inventory: FakeInventory, server: FakeInspectServer, mode: str = "tx") -> None: @@ -113,12 +113,12 @@ def process(self, context: ProcessingContext, params: _Params) -> ProcessorResul def test_construction_is_lazy_and_injection_matches_configuration() -> None: - options = ApplyOptions(sync="none", discovery_timeout=60) + options = ApplyOptions(sync="none", topology_ready_timeout=60) naming = NamingScheme(endpoint_label="{device.label}-{endpoint.index:02d}") - injected = BlueprintEngine( + injected = ProvisioningEngine( _ExplodingApp(), processors={"example-org.simple": SimpleVideoProcessor}, options=options, naming=naming ) - configured = BlueprintEngine(_ExplodingApp()) + configured = ProvisioningEngine(_ExplodingApp()) configured.register_processor("example-org.simple", SimpleVideoProcessor) configured.configure(options=options) configured.configure(naming=naming) @@ -133,19 +133,19 @@ def test_construction_is_lazy_and_injection_matches_configuration() -> None: configured.register_processor("example-org.simple", SimpleVideoProcessor) with pytest.raises(TypeError): configured.configure(options={"sync": "none"}) # type: ignore[arg-type] - assert "example-org.simple" not in BlueprintEngine(_ExplodingApp()).registry + assert "example-org.simple" not in ProvisioningEngine(_ExplodingApp()).registry def test_apply_has_the_plan_signature_plus_dry_run() -> None: - plan_parameters = list(inspect.signature(BlueprintEngine.plan).parameters.values()) - apply_parameters = list(inspect.signature(BlueprintEngine.apply).parameters.values()) + plan_parameters = list(inspect.signature(ProvisioningEngine.plan).parameters.values()) + apply_parameters = list(inspect.signature(ProvisioningEngine.apply).parameters.values()) assert apply_parameters[:-1] == plan_parameters assert (apply_parameters[-1].name, apply_parameters[-1].default) == ("dry_run", False) - assert inspect.signature(BlueprintPlan.apply).parameters["dry_run"].default is False + assert inspect.signature(ProvisioningPlan.apply).parameters["dry_run"].default is False -def test_normal_app_import_does_not_load_blueprints() -> None: - code = "import sys, videoipath_automation_tool; print('videoipath_automation_tool.blueprints' in sys.modules, 'yaml' in sys.modules)" +def test_normal_app_import_does_not_load_provisioning() -> None: + code = "import sys, videoipath_automation_tool; print('videoipath_automation_tool.provisioning' in sys.modules, 'yaml' in sys.modules)" output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout.split() assert output == ["False", "False"] @@ -154,7 +154,7 @@ def test_normal_app_import_does_not_load_blueprints() -> None: def test_planning_is_read_only( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" @@ -162,7 +162,14 @@ def test_planning_is_read_only( assert plan.fully_resolved and plan.has_changes assert inventory.writes == [] and server.writes == [] assert app.inspect._snapshot is None - assert [phase.status for phase in plan.phases] == ["planned", "skipped", "no_change", "planned", "skipped"] + assert [phase.status for phase in plan.phases] == [ + "planned", + "skipped", + "no_change", + "skipped", + "planned", + "skipped", + ] assert plan.phase("inventory").operations[0].changes[0].model_dump() == { "field": "label", "before": "old", @@ -171,19 +178,29 @@ def test_planning_is_read_only( "sensitive": False, } assert [b.port_id for b in plan.interface_bindings] == ["device1.dev.0.P1", "device1.dev.0.P2"] + assert [b.key for b in plan.interface_bindings] == ["stream-a", "stream-b"] + + print(plan.summary()) + assert inventory.writes == [] and server.writes == [] and app.inspect._snapshot is None + result = plan.apply() + assert result.status == "succeeded" and result.ok + assert result.dry_run is False def test_inventory_only_scope_never_touches_inspect(inventory: FakeInventory) -> None: inventory.seed("device1", label="device-a") app = SimpleNamespace(inventory=inventory) - document = {**MATROX, "topology": {"default": {"vertex_processor": {"processor_type": "example-org.missing"}}}} - result = BlueprintEngine(app).apply(_device(), Blueprint.from_dict(document), scope="inventory") + document = { + **MATROX, + "defaults": {**MATROX["defaults"], "topology": {"vertex_processor": {"processor_type": "example-org.missing"}}}, + } + result = ProvisioningEngine(app).apply(_device(), Blueprint.from_dict(document), scope="inventory") assert result.status == "succeeded" and result.phase("topology").status == "skipped" assert inventory.devices["device1"].configuration.config.customSettings.port == 8080 def test_apply_then_reapply_converges( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) result = engine.apply(_device(), _blueprint()) @@ -203,20 +220,26 @@ def test_both_usage_styles_execute_the_same_workflow() -> None: for style in ("direct", "two-step"): inventory, server = FakeInventory(), FakeInspectServer() _existing(inventory, server) - engine = BlueprintEngine(SimpleNamespace(inventory=inventory, inspect=make_inspect_app(server))) + engine = ProvisioningEngine(SimpleNamespace(inventory=inventory, inspect=make_inspect_app(server))) if style == "direct": - result = engine.apply(_device(), _blueprint(), topology_variant="default") + result = engine.apply(_device(), _blueprint(), variants=()) else: - result = engine.plan(_device(), _blueprint(), topology_variant="default").apply() + result = engine.plan(_device(), _blueprint(), variants=()).apply() outcomes.append((result.model_dump(), server.writes, inventory.writes)) assert outcomes[0] == outcomes[1] def test_update_preserves_unmanaged_fields_and_redacts_secrets( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) - document = {**MATROX, "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}}} + document = { + **MATROX, + "defaults": { + **MATROX["defaults"], + "inventory": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}, + }, + } device = _device( credentials=Credentials(username="test-user", password="test-password"), alternative_addresses=[ @@ -265,7 +288,7 @@ def test_update_preserves_unmanaged_fields_and_redacts_secrets( def test_masked_secrets_on_read_do_not_cause_changes( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) device = _device( @@ -294,18 +317,13 @@ def test_masked_secrets_on_read_do_not_cause_changes( assert cinfo.altAddressesWithAuth[0]["authentication"]["password"] == "alt-password" -def test_api_key_is_redacted_in_the_plan(engine: BlueprintEngine) -> None: +def test_api_key_is_redacted_in_the_plan(engine: ProvisioningEngine) -> None: secret = "synthetic-key-value" document = { "schema_version": 1, - "inventory": { - "default": { - "driver_id": "com.nevion.spg9000-0.1.0", - "custom_settings": {"x_api_key": secret}, - } - }, + "defaults": {"inventory": {"driver_id": "com.nevion.spg9000-0.1.0", "custom_settings": {"x_api_key": secret}}}, } - device = BlueprintDevice(key="key-a", label="device-a", management_address="192.0.2.40") + device = ProvisioningDevice(key="key-a", label="device-a", management_address="192.0.2.40") plan = engine.plan(device, Blueprint.from_dict(document), scope="inventory") change = next( item for item in plan.phase("inventory").operations[0].changes if item.field == "custom_settings.x_api_key" @@ -315,34 +333,39 @@ def test_api_key_is_redacted_in_the_plan(engine: BlueprintEngine) -> None: assert secret not in plan.model_dump_json() -def test_inventory_binding_rules(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_inventory_binding_rules( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: inventory.seed("device1", label="device-a", address="192.0.2.10") - with pytest.raises(BlueprintTargetError, match="'device9' was not found"): + with pytest.raises(ProvisioningTargetError, match="'device9' was not found"): engine.plan(_device(inventory_id="device9"), _blueprint(), scope="inventory") - new = BlueprintDevice(key="key-b", label="device-a", management_address="192.0.2.10") - with pytest.raises(BlueprintTargetError, match="used by device1.*Bind the existing record"): + new = ProvisioningDevice(key="key-b", label="device-a", management_address="192.0.2.10") + with pytest.raises(ProvisioningTargetError, match="used by device1.*Bind the existing record"): engine.plan(new, _blueprint(), scope="inventory") assert inventory.address_lookups == 1 - with pytest.raises(BlueprintTargetError, match="requires 'management_address'"): - engine.plan(BlueprintDevice(key="key-b", label="device-b"), _blueprint(), scope="inventory") - other_driver = {**MATROX, "inventory": {"default": {"driver_id": "com.nevion.NMOS-0.1.0"}}} - with pytest.raises(BlueprintCapabilityError, match="Driver migration"): + with pytest.raises(ProvisioningTargetError, match="requires 'management_address'"): + engine.plan(ProvisioningDevice(key="key-b", label="device-b"), _blueprint(), scope="inventory") + other_driver = {**MATROX, "defaults": {**MATROX["defaults"], "inventory": {"driver_id": "com.nevion.NMOS-0.1.0"}}} + with pytest.raises(ProvisioningCapabilityError, match="Driver migration"): engine.plan(_device(), Blueprint.from_dict(other_driver), scope="inventory") -def test_snmp_references_resolve_exactly(engine: BlueprintEngine, inventory: FakeInventory) -> None: +def test_snmp_references_resolve_exactly(engine: ProvisioningEngine, inventory: FakeInventory) -> None: inventory.seed("device1", label="device-a") def plan_with(reference: Any) -> Any: - document = {**MATROX, "inventory": {"default": {"driver_id": NMOS, "snmp": {"configuration": reference}}}} + document = { + **MATROX, + "defaults": {**MATROX["defaults"], "inventory": {"driver_id": NMOS, "snmp": {"configuration": reference}}}, + } return engine.plan(_device(), Blueprint.from_dict(document), scope="inventory") assert plan_with("snmp-a").phase("inventory").operations[0].changes[0].after == "snmp-1" assert plan_with({"id": "default"}).phase("inventory").status == "no_change" - with pytest.raises(BlueprintTargetError, match="No SNMP configuration"): + with pytest.raises(ProvisioningTargetError, match="No SNMP configuration"): plan_with("missing") inventory.snmp["snmp-2"] = "snmp-a" - with pytest.raises(BlueprintTargetError, match="ambiguous"): + with pytest.raises(ProvisioningTargetError, match="ambiguous"): plan_with("snmp-a") @@ -350,9 +373,9 @@ def plan_with(reference: Any) -> Any: def test_create_discover_and_configure( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") plan = engine.plan(device, _blueprint()) assert not plan.fully_resolved and plan.phase("topology").status == "deferred" assert "materialized during apply()" in plan.summary() @@ -368,12 +391,18 @@ def test_create_discover_and_configure( def _topology_affecting_document() -> Blueprint: return Blueprint.from_dict( - {**MATROX, "inventory": {"default": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}}} + { + **MATROX, + "defaults": { + **MATROX["defaults"], + "inventory": {"driver_id": NMOS, "custom_settings": {"disable_rx_sdp": True}}, + }, + } ) def test_topology_affecting_update_stops_for_replan( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) document = _topology_affecting_document() @@ -387,9 +416,10 @@ def test_topology_affecting_update_stops_for_replan( result = plan.apply() assert result.status == "partial" and result.replan_required and not result.ok assert result.phase("inventory").status == "completed" and result.verification == "confirmed" - assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { - "deferred" - } + assert { + result.phase(name).status + for name in ("inventory_readiness", "topology_sync", "discovery", "topology", "module_tags") + } == {"deferred"} assert inventory.writes == [("update", "device1")] and server.writes == [] assert inventory.devices["device1"].configuration.config.customSettings.disable_rx_sdp is True @@ -400,51 +430,53 @@ def test_topology_affecting_update_stops_for_replan( assert "update_topology" in [name for name, _ in server.writes] -def test_topology_affecting_update_dry_run_requires_replan( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer +def test_topology_affecting_update_plan_defers_topology_without_writes( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) - result = engine.apply(_device(), _topology_affecting_document(), dry_run=True) - assert result.status == "planned" and result.replan_required and result.dry_run - assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { - "deferred" - } - assert "plan again" in (result.phase("topology").message or "") + plan = engine.plan(_device(), _topology_affecting_document()) + assert plan.has_changes and not plan.fully_resolved + assert plan.phase("inventory").status == "planned" + assert { + plan.phase(name).status + for name in ("inventory_readiness", "topology_sync", "discovery", "topology", "module_tags") + } == {"deferred"} + assert "plan again" in (plan.phase("topology").reason or "") assert inventory.writes == [] and server.writes == [] -def test_discovery_timeout_reports_created_id(engine: BlueprintEngine, server: FakeInspectServer) -> None: - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") - with pytest.raises(BlueprintApplyError) as info: - engine.apply(device, _blueprint(), options=ApplyOptions(discovery_timeout=3, poll_interval=1)) +def test_topology_ready_timeout_reports_created_id(engine: ProvisioningEngine, server: FakeInspectServer) -> None: + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(device, _blueprint(), options=ApplyOptions(topology_ready_timeout=3, poll_interval=1)) result = info.value.result assert result.status == "partial" and result.inventory_id == "device100" assert result.phase("inventory").status == "completed" - assert result.phase("discovery").status == "failed" - assert "not ready after 3s" in result.phase("discovery").message - assert [name for name, _ in server.writes].count("add_devices") == 4 + assert result.phase("topology_sync").status == "failed" + assert "not ready after 3s" in result.phase("topology_sync").message + assert [name for name, _ in server.writes].count("add_devices") == 3 def test_permanent_errors_during_discovery_fail_without_waiting( - engine: BlueprintEngine, server: FakeInspectServer + engine: ProvisioningEngine, server: FakeInspectServer ) -> None: - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") server.discoverable["device100"] = matrox_layout("device100", "tx") wrong_mode = {"processor_type": "matrox.convertip.default", "params": {"mode": "rx"}} - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(device, _blueprint(vertex_processor=wrong_mode)) result = info.value.result assert result.status == "partial" and result.inventory_id == "device100" - assert "mode 'rx' was requested" in result.phase("topology").message - assert "not ready" not in result.phase("topology").message + assert "mode 'rx' was requested" in result.phase("discovery").message + assert "not ready" not in result.phase("discovery").message assert engine._clock() == 0.0 def test_interface_miss_is_final_only_once_ports_exist( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) - with pytest.raises(BlueprintTargetError, match="no IP port") as info: + with pytest.raises(ProvisioningTargetError, match="no IP port") as info: engine.plan(_device(), _blueprint(ip_vertex_mapping={"stream-a": ["missing"]})) assert not isinstance(info.value, TopologyNotReadyError) empty = ScopeData(device_id="device1", device=DeviceRecord(id="device1")) @@ -452,33 +484,35 @@ def test_interface_miss_is_final_only_once_ports_exist( resolve_interfaces({"stream-a": ["P1"]}, empty, None) -def test_address_conflicts_use_one_lookup_for_all_addresses(engine: BlueprintEngine, inventory: FakeInventory) -> None: +def test_address_conflicts_use_one_lookup_for_all_addresses( + engine: ProvisioningEngine, inventory: FakeInventory +) -> None: inventory.seed("device1", label="device-a", address="Device-A.example") inventory.seed("device2", label="device-b", address="2001:db8:0:0::1") - new = BlueprintDevice( + new = ProvisioningDevice( key="key-c", label="device-c", management_address="device-a.example", alternative_addresses=["2001:db8::1"] ) - with pytest.raises(BlueprintTargetError, match="'device-a.example' is used by device1; address '2001:db8::1'"): + with pytest.raises(ProvisioningTargetError, match="'device-a.example' is used by device1; address '2001:db8::1'"): engine.plan(new, _blueprint(), scope="inventory") assert inventory.address_lookups == 1 -def test_address_read_failure_fails_planning(engine: BlueprintEngine, inventory: FakeInventory) -> None: +def test_address_read_failure_fails_planning(engine: ProvisioningEngine, inventory: FakeInventory) -> None: inventory.fail_address_read = True - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") - with pytest.raises(BlueprintError, match="Could not read Inventory addresses") as info: + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(ProvisioningError, match="Could not read Inventory addresses") as info: engine.plan(device, _blueprint(), scope="inventory") - assert type(info.value) is BlueprintError + assert type(info.value) is ProvisioningError -def test_sync_policies(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_sync_policies(engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: inventory.seed("device1", label="device-a", custom={"port": 8080}) - with pytest.raises(BlueprintTargetError, match="not in the topology"): + with pytest.raises(ProvisioningTargetError, match="not in the topology"): engine.plan(_device(), _blueprint(), options=ApplyOptions(sync="none")) server.install(matrox_layout("device1", "tx")) server.sync["device1"] = {"add": {}, "update": {"x": 1}, "remove": {}} - with pytest.raises(BlueprintCapabilityError, match="add_only' is insufficient"): + with pytest.raises(ProvisioningCapabilityError, match="add_only' is insufficient"): engine.plan(_device(), _blueprint()) warned = engine.plan(_device(), _blueprint(), options=ApplyOptions(sync="none")) assert warned.fully_resolved and warned.diagnostics[0].code == "topology.sync_pending" @@ -488,22 +522,22 @@ def test_sync_policies(engine: BlueprintEngine, inventory: FakeInventory, server def test_sync_lookup_failure_fails_planning( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) server.fail_sync_info = ConnectionError("sync lookup failed") - with pytest.raises(BlueprintError, match="synchronization status") as info: + with pytest.raises(ProvisioningError, match="synchronization status") as info: engine.plan(_device(), _blueprint()) - assert type(info.value) is BlueprintError + assert type(info.value) is ProvisioningError def test_sync_failure_is_not_retried( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) server.sync["device1"] = {"add": {"modules": 1}, "update": {}, "remove": {}} server.fail_sync = True - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(_device(), _blueprint()) result = info.value.result assert result.status == "failed" @@ -513,59 +547,61 @@ def test_sync_failure_is_not_retried( def test_device_form_lookup_failure_fails_planning( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) del server.device_forms["device1"] - with pytest.raises(BlueprintError, match="Inspect edit form") as info: + with pytest.raises(ProvisioningError, match="Inspect edit form") as info: engine.plan(_device(), _blueprint()) - assert type(info.value) is BlueprintError + assert type(info.value) is ProvisioningError # --- Conflicts and staged edits --- -def test_stale_plans_are_rejected(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_stale_plans_are_rejected( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" plan = engine.plan(_device(), _blueprint()) inventory.devices["device1"].configuration.config.desc.label = "edited-elsewhere" - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: plan.apply() - assert info.value.result.status == "failed" and isinstance(info.value.__cause__, BlueprintConflictError) + assert info.value.result.status == "failed" and isinstance(info.value.__cause__, ProvisioningConflictError) assert inventory.writes == [] inventory.devices["device1"].configuration.config.desc.label = "device-a" plan = engine.plan(_device(), _blueprint()) server.vertex_forms["device1.0.vs.v"]["fields"]["desc"] = "edited-elsewhere" - with pytest.raises(BlueprintApplyError, match="changed since planning"): + with pytest.raises(ProvisioningApplyError, match="changed since planning"): plan.apply() assert server.writes == [] def test_inventory_update_rechecks_label_conflicts( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" plan = engine.plan(_device(), _blueprint()) assert plan.fully_resolved and plan.phase("inventory").status == "planned" inventory.seed("device2", label="device-a") - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: plan.apply() assert info.value.result.status == "failed" - assert isinstance(info.value.__cause__, BlueprintTargetError) + assert isinstance(info.value.__cause__, ProvisioningTargetError) assert inventory.writes == [] def test_staged_edits_overlap_is_rejected_and_unrelated_edits_survive( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: _existing(inventory, server) app.inspect._snapshot = InspectSnapshot(fetcher=server, device_items=server.get_device_skeleton()) app.inspect._snapshot.stage_edit("vertex", "device1.0.vs.v", "label", "pending") app.inspect._snapshot.stage_edit("vertex", "device1.0.P1.in", "label", "unrelated") - with pytest.raises(BlueprintApplyError, match="Pending uncommitted Inspect edits overlap"): + with pytest.raises(ProvisioningApplyError, match="Pending uncommitted Inspect edits overlap"): engine.apply(_device(), _blueprint()) app.inspect._snapshot.clear_staged(kind="vertex", entity_id="device1.0.vs.v") assert engine.apply(_device(), _blueprint()).status == "succeeded" @@ -576,15 +612,15 @@ def test_staged_edits_overlap_is_rejected_and_unrelated_edits_survive( @pytest.mark.parametrize( - ("error", "status"), [(ConnectionError("timeout"), "unknown"), (BlueprintConflictError("x"), "partial")] + ("error", "status"), [(ConnectionError("timeout"), "unknown"), (ProvisioningConflictError("x"), "partial")] ) def test_commit_failure_outcomes( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, error: Exception, status: str + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, error: Exception, status: str ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" server.fail_commit = error - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(_device(), _blueprint()) result = info.value.result assert result.status == status and result.phase("inventory").status == "completed" @@ -593,25 +629,25 @@ def test_commit_failure_outcomes( def test_inventory_write_unknown_outcome( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" inventory.fail_next_write = TimeoutError("read timed out") - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(_device(), _blueprint()) assert info.value.result.status == "unknown" and info.value.result.phase("topology").status == "not_run" def test_inventory_preread_failure_is_a_known_rejection( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" inventory.fail_next_write = InventoryWriteNotAppliedError( "Failed to retrieve existing device configuration from Inventory: timed out", operation="update" ) - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(_device(), _blueprint()) result = info.value.result assert result.status == "failed" and result.phase("inventory").status == "failed" @@ -621,23 +657,23 @@ def test_inventory_preread_failure_is_a_known_rejection( @pytest.mark.parametrize(("operation", "status"), [("add", "failed"), ("update", "unknown")]) def test_inventory_create_rejection_outcomes( - engine: BlueprintEngine, inventory: FakeInventory, operation: str, status: str + engine: ProvisioningEngine, inventory: FakeInventory, operation: str, status: str ) -> None: # A rejected add is known; a rejected follow-up update means the record may already exist. inventory.fail_next_write = InventoryWriteNotAppliedError("rejected", operation=operation) # type: ignore[arg-type] - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") - with pytest.raises(BlueprintApplyError) as info: + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(ProvisioningApplyError) as info: engine.apply(device, _blueprint(), scope="inventory") assert info.value.result.status == status def test_untyped_inventory_value_error_is_an_unknown_outcome( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" inventory.fail_next_write = ValueError("Failed to update device in VideoIPath-Inventory.") - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(_device(), _blueprint()) assert info.value.result.status == "unknown" @@ -659,19 +695,19 @@ def _chassis(server: FakeInspectServer, inventory: FakeInventory) -> None: def test_module_target_scope_and_tags( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) document = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "module": {"tags": {"add": ["new"], "remove": ["old"]}}, "vertex_processor": {"processor_type": "matrox.convertip.default"}, } }, } - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1"), @@ -704,61 +740,63 @@ def _mark_endpoint(server: FakeInspectServer, vertex_id: str) -> None: def test_sibling_endpoint_label_change_rejects_the_plan( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) sibling = "device1.2.vs.v" _mark_endpoint(server, sibling) server.vertex_forms[sibling]["fields"]["label"] = "sibling-endpoint" - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1"), ) document = { "schema_version": 1, - "topology": {"default": {"vertex_processor": {"processor_type": "matrox.convertip.default"}}}, + "defaults": {"topology": {"vertex_processor": {"processor_type": "matrox.convertip.default"}}}, } plan = engine.plan(module, Blueprint.from_dict(document)) assert plan.fully_resolved server.vertex_forms[sibling]["fields"]["label"] = "renamed-endpoint" - with pytest.raises(BlueprintApplyError, match="changed since planning") as info: + with pytest.raises(ProvisioningApplyError, match="changed since planning") as info: plan.apply() - assert isinstance(info.value.__cause__, BlueprintConflictError) + assert isinstance(info.value.__cause__, ProvisioningConflictError) assert server.writes == [] -def test_module_target_rejections(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_module_target_rejections( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: _chassis(server, inventory) - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) - device_settings = {"schema_version": 1, "topology": {"default": {"device": {"icon_type": "monitor"}}}} - with pytest.raises(BlueprintTargetError, match="cannot be applied to a module target"): + device_settings = {"schema_version": 1, "defaults": {"topology": {"device": {"icon_type": "monitor"}}}} + with pytest.raises(ProvisioningTargetError, match="cannot be applied to a module target"): engine.plan(module, Blueprint.from_dict(device_settings)) missing = module.model_copy(update={"topology": ModuleTarget(device_id="device1", module_id="device2.dev.1")}) - with pytest.raises(BlueprintTargetError, match="does not belong"): - engine.plan(missing, Blueprint.from_dict({"schema_version": 1, "topology": {"default": {}}})) + with pytest.raises(ProvisioningTargetError, match="does not belong"): + engine.plan(missing, Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": {}}})) def test_module_tag_failure_is_a_partial_result( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) server.fail_tag = "old" document = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "module": {"tags": ["keep", "new"]}, "vertex_processor": {"processor_type": "matrox.convertip.default"}, } }, } - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: engine.apply(module, Blueprint.from_dict(document)) result = info.value.result assert result.status == "partial" @@ -773,11 +811,11 @@ def test_module_tag_failure_is_a_partial_result( def test_label_collisions_and_overrides( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) colliding = NamingScheme(endpoint_label="{device.label}-{endpoint.direction}") - with pytest.raises(BlueprintValidationError, match="Endpoint label collision"): + with pytest.raises(ProvisioningValidationError, match="Endpoint label collision"): engine.plan(_device(), _blueprint(), naming=colliding) assert engine.plan( _device(), _blueprint(), naming=colliding, options=ApplyOptions(naming_collisions="allow") @@ -789,11 +827,13 @@ def test_label_collisions_and_overrides( assert changes["device1.0.as.v"]["label"].after == "custom-audio" assert changes["device1.0.as.v"]["label"].source.startswith("override:") - with pytest.raises(BlueprintTargetError, match="must match exactly one vertex"): + with pytest.raises(ProvisioningTargetError, match="must match exactly one vertex"): engine.plan(_device(), _blueprint(vertices=[{"factory_label": "P1", "fields": {"active": False}}])) -def test_interface_mapping_rules(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_interface_mapping_rules( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: _existing(inventory, server) mapping = {"a": ["X{module.position}", "P{module.position}"], "b": ["P2"]} plan = engine.plan(_device(module_position="1"), _blueprint(ip_vertex_mapping=mapping)) @@ -801,9 +841,9 @@ def test_interface_mapping_rules(engine: BlueprintEngine, inventory: FakeInvento ("a", "P1", "device1.0.P1.out"), ("b", "P2", "device1.0.P2.out"), ] - with pytest.raises(BlueprintTargetError, match="no IP port .* matches 'X1', 'X2'"): + with pytest.raises(ProvisioningTargetError, match="no IP port .* matches 'X1', 'X2'"): engine.plan(_device(), _blueprint(ip_vertex_mapping={"a": ["X1", "X2"]})) - with pytest.raises(BlueprintValidationError, match="requires BlueprintDevice.module_position"): + with pytest.raises(ProvisioningValidationError, match="requires ProvisioningDevice.module_position"): engine.plan(_device(), _blueprint(ip_vertex_mapping={"a": ["P{module.position}"]})) @@ -811,11 +851,11 @@ def test_custom_processor_and_naming_change_rename_the_same_ids( inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: _existing(inventory, server) - engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) + engine = ProvisioningEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) document = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "vertex_processor": { "processor_type": "example-org.simple-video", "params": {"factory_label": "Video Sender"}, @@ -833,22 +873,22 @@ def test_custom_processor_and_naming_change_rename_the_same_ids( assert server.vertex_forms["device1.0.vs.v"]["fields"]["label"] == "device-a-video" -def test_topology_only_virtual_device(engine: BlueprintEngine, server: FakeInspectServer) -> None: +def test_topology_only_virtual_device(engine: ProvisioningEngine, server: FakeInspectServer) -> None: layout = matrox_layout("virtual.1", "tx") layout["node"] = {"_id": "virtual-1", "modules": {}} server.install(layout) server.nodes["virtual-1"] = server.nodes.pop("virtual.1") - device = BlueprintDevice(key="virtual-a", label="virtual-a", topology=DeviceTarget(device_id="virtual.1")) - document = {"schema_version": 1, "topology": {"default": {"device": {"icon_type": "server"}}}} + device = ProvisioningDevice(key="virtual-a", label="virtual-a", topology=DeviceTarget(device_id="virtual.1")) + document = {"schema_version": 1, "defaults": {"topology": {"device": {"icon_type": "server"}}}} result = engine.apply(device, Blueprint.from_dict(document)) assert result.status == "succeeded" assert server.device_forms["virtual.1"]["iconType"] == "server" assert server.device_forms["virtual.1"]["descriptor"]["label"] == "virtual-a" -def test_from_inventory_and_engine_validation(engine: BlueprintEngine, inventory: FakeInventory) -> None: +def test_from_inventory_and_engine_validation(engine: ProvisioningEngine, inventory: FakeInventory) -> None: record = inventory.seed("device1", label="device-a") - device = BlueprintDevice.from_inventory(record) + device = ProvisioningDevice.from_inventory(record) assert (device.key, device.label, device.inventory_id, device.management_address) == ( "device1", "device-a", @@ -856,8 +896,8 @@ def test_from_inventory_and_engine_validation(engine: BlueprintEngine, inventory None, ) engine.validate(_blueprint()) - unknown = {"schema_version": 1, "topology": {"default": {"vertex_processor": {"processor_type": "example-org.x"}}}} - with pytest.raises(BlueprintValidationError, match="not registered"): + unknown = {"schema_version": 1, "defaults": {"topology": {"vertex_processor": {"processor_type": "example-org.x"}}}} + with pytest.raises(ProvisioningValidationError, match="not registered"): engine.validate(Blueprint.from_dict(unknown)) @@ -902,11 +942,11 @@ def process(self, context: ProcessingContext, params: _ModeParams) -> Any: @pytest.mark.parametrize( ("mode", "error", "message"), [ - ("sibling", BlueprintTargetError, "outside module 'device1.dev.1'"), - ("device-patch", BlueprintTargetError, "device-wide changes for a module target"), + ("sibling", ProvisioningTargetError, "outside module 'device1.dev.1'"), + ("device-patch", ProvisioningTargetError, "device-wide changes for a module target"), ("conflict", ProcessorInputError, "conflicting 'label' values"), ("bypass", ProcessorInputError, "invalid result"), - ("sdp-on-ip", BlueprintCapabilityError, "only supported on codec vertices"), + ("sdp-on-ip", ProvisioningCapabilityError, "only supported on codec vertices"), ("wrong-type", ProcessorInputError, "must return a ProcessorResult"), ("crash", ProcessorInputError, "failed: RuntimeError: boom"), ], @@ -915,12 +955,12 @@ def test_processor_output_is_validated( inventory: FakeInventory, server: FakeInspectServer, app: Any, mode: str, error: type, message: str ) -> None: _chassis(server, inventory) - engine = BlueprintEngine(app, processors={"example-org.bad": MisbehavingProcessor}) + engine = ProvisioningEngine(app, processors={"example-org.bad": MisbehavingProcessor}) document = { "schema_version": 1, - "topology": {"default": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": mode}}}}, + "defaults": {"topology": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": mode}}}}, } - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) with pytest.raises(error, match=message): @@ -932,33 +972,33 @@ def test_identical_duplicate_edits_are_deduplicated( inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: _chassis(server, inventory) - engine = BlueprintEngine(app, processors={"example-org.bad": MisbehavingProcessor}) + engine = ProvisioningEngine(app, processors={"example-org.bad": MisbehavingProcessor}) document = { "schema_version": 1, - "topology": { - "default": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": "duplicate"}}} + "defaults": { + "topology": {"vertex_processor": {"processor_type": "example-org.bad", "params": {"mode": "duplicate"}}} }, } - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) assert engine.plan(module, Blueprint.from_dict(document)).phase("topology").status == "no_change" def test_module_tag_baseline_conflict_after_topology_commit( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) document = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "module": {"tags": ["keep", "new"]}, "vertex_processor": {"processor_type": "matrox.convertip.default"}, } }, } - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) plan = engine.plan(module, Blueprint.from_dict(document)) @@ -970,7 +1010,7 @@ def commit_then_retag(delta: Any) -> Any: return response server.update_topology = commit_then_retag # type: ignore[method-assign] - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: plan.apply() result = info.value.result assert (result.status, result.phase("topology").status, result.phase("module_tags").status) == ( @@ -982,33 +1022,33 @@ def commit_then_retag(delta: Any) -> Any: def test_module_tags_without_local_distinction_allow_only_additions( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) server.nodes["device1"]["modules"]["device1.dev.1"]["tagsInfo"] = {"assigned": {"all": ["inherited"]}} - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) def plan_with(tags: Any) -> Any: return engine.plan( - module, Blueprint.from_dict({"schema_version": 1, "topology": {"default": {"module": {"tags": tags}}}}) + module, Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": {"module": {"tags": tags}}}}) ) - with pytest.raises(BlueprintCapabilityError, match="cannot be distinguished"): + with pytest.raises(ProvisioningCapabilityError, match="cannot be distinguished"): plan_with(["keep"]) - with pytest.raises(BlueprintCapabilityError, match="cannot be distinguished"): + with pytest.raises(ProvisioningCapabilityError, match="cannot be distinguished"): plan_with({"remove": ["inherited"]}) assert [op.action for op in plan_with({"add": ["new"]}).phase("module_tags").operations] == ["assign_tag"] def test_module_with_own_inventory_record_never_touches_the_parent( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _chassis(server, inventory) inventory.seed("device5", label="module-a", address="192.0.2.55", custom={"port": 80}) server.sync["device1"] = {"add": {"x": 1}, "update": {}, "remove": {}} - module = BlueprintDevice( + module = ProvisioningDevice( key="module-a", label="module-a", inventory_id="device5", @@ -1032,11 +1072,43 @@ def test_module_with_own_inventory_record_never_touches_the_parent( assert delta["replaceDevices"] == {} and all(v.startswith("device1.1.") for v in delta["replaceVertices"]) +# --- Planning deferred work and module tags --- + + +def test_plan_reports_deferred_work_for_creation_without_writes( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") + plan = engine.plan(device, _blueprint()) + assert plan.has_changes and not plan.fully_resolved and plan.inventory_id is None + assert plan.phase("inventory").status == "planned" + assert plan.phase("inventory").operations[0].action == "create" + assert { + plan.phase(name).status + for name in ("inventory_readiness", "topology_sync", "discovery", "topology", "module_tags") + } == {"deferred"} + assert inventory.writes == [] and server.writes == [] + + +def test_plan_module_tags_without_writes( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _chassis(server, inventory) + document = {"schema_version": 1, "defaults": {"topology": {"module": {"tags": ["keep", "new"]}}}} + module = ProvisioningDevice( + key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") + ) + plan = engine.plan(module, Blueprint.from_dict(document)) + assert plan.phase("module_tags").status == "planned" + assert [op.action for op in plan.phase("module_tags").operations] == ["assign_tag", "unassign_tag"] + assert inventory.writes == [] and server.writes == [] + + # --- Dry run --- def test_dry_run_checks_everything_but_writes_nothing( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, app: Any ) -> None: _existing(inventory, server) inventory.devices["device1"].configuration.config.desc.label = "old" @@ -1051,8 +1123,9 @@ def test_dry_run_checks_everything_but_writes_nothing( ) assert [(p.name, p.status) for p in preview.phases] == [ ("inventory", "planned"), - ("discovery", "skipped"), + ("inventory_readiness", "skipped"), ("topology_sync", "no_change"), + ("discovery", "skipped"), ("topology", "planned"), ("module_tags", "skipped"), ("verification", "skipped"), @@ -1067,34 +1140,35 @@ def test_dry_run_checks_everything_but_writes_nothing( def test_dry_run_detects_stale_plans( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: _existing(inventory, server) plan = engine.plan(_device(), _blueprint()) server.vertex_forms["device1.0.vs.v"]["fields"]["desc"] = "edited-elsewhere" - with pytest.raises(BlueprintApplyError) as info: + with pytest.raises(ProvisioningApplyError) as info: plan.apply(dry_run=True) assert info.value.result.status == "failed" and info.value.result.dry_run assert server.writes == [] def test_dry_run_reports_deferred_work_for_creation( - engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer ) -> None: - device = BlueprintDevice(key="key-new", label="device-new", management_address="192.0.2.50") + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") result = engine.apply(device, _blueprint(), dry_run=True) assert result.status == "planned" and result.inventory_id is None assert result.phase("inventory").operations[0].action == "create" - assert {result.phase(name).status for name in ("discovery", "topology_sync", "topology", "module_tags")} == { - "deferred" - } + assert { + result.phase(name).status + for name in ("inventory_readiness", "topology_sync", "discovery", "topology", "module_tags") + } == {"deferred"} assert inventory.writes == [] and server.writes == [] -def test_dry_run_module_tags(engine: BlueprintEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: +def test_dry_run_module_tags(engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer) -> None: _chassis(server, inventory) - document = {"schema_version": 1, "topology": {"default": {"module": {"tags": ["keep", "new"]}}}} - module = BlueprintDevice( + document = {"schema_version": 1, "defaults": {"topology": {"module": {"tags": ["keep", "new"]}}}} + module = ProvisioningDevice( key="module-a", label="module-a", topology=ModuleTarget(device_id="device1", module_id="device1.dev.1") ) result = engine.apply(module, Blueprint.from_dict(document), dry_run=True) @@ -1130,10 +1204,10 @@ def process(self, context: ProcessingContext, params: _KindFieldParams) -> Proce def test_kind_specific_fields_round_trip(inventory: FakeInventory, server: FakeInspectServer, app: Any) -> None: _existing(inventory, server) - engine = BlueprintEngine(app, processors={"example-org.kind-fields": KindFieldProcessor}) + engine = ProvisioningEngine(app, processors={"example-org.kind-fields": KindFieldProcessor}) document = { "schema_version": 1, - "topology": {"default": {"vertex_processor": {"processor_type": "example-org.kind-fields"}}}, + "defaults": {"topology": {"vertex_processor": {"processor_type": "example-org.kind-fields"}}}, } result = engine.apply(_device(), Blueprint.from_dict(document), scope="topology") assert result.status == "succeeded" and result.verification == "confirmed" @@ -1146,18 +1220,20 @@ def test_kind_specific_fields_round_trip(inventory: FakeInventory, server: FakeI wrong = { "schema_version": 1, - "topology": { - "default": { + "defaults": { + "topology": { "vertex_processor": {"processor_type": "example-org.kind-fields", "params": {"ports_on_ip": True}} } }, } - with pytest.raises(BlueprintCapabilityError, match="'main_destination_port' is only supported on codec vertices"): + with pytest.raises( + ProvisioningCapabilityError, match="'main_destination_port' is only supported on codec vertices" + ): engine.plan(_device(), Blueprint.from_dict(wrong), scope="topology") def test_engine_schema_lists_only_registered_processors(app: Any) -> None: - engine = BlueprintEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) - spec = engine.json_schema()["$defs"]["VertexProcessorSpec"] + engine = ProvisioningEngine(app, processors={"example-org.simple-video": SimpleVideoProcessor}) + spec = engine.json_schema()["$defs"]["VertexProcessorSpecTemplate"] assert spec["properties"]["processor_type"]["enum"] == ["example-org.simple-video", "matrox.convertip.default"] - assert "enum" not in published_json_schema()["$defs"]["VertexProcessorSpec"]["properties"]["processor_type"] + assert "enum" not in published_json_schema()["$defs"]["VertexProcessorSpecTemplate"]["properties"]["processor_type"] diff --git a/tests/provisioning/test_live_regressions.py b/tests/provisioning/test_live_regressions.py new file mode 100644 index 0000000..04452ff --- /dev/null +++ b/tests/provisioning/test_live_regressions.py @@ -0,0 +1,91 @@ +"""Anonymized regressions for server behavior first observed by the live suite.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from videoipath_automation_tool.apps.inspect.errors import InspectEntityNotFoundError +from videoipath_automation_tool.provisioning import Blueprint, DeviceTarget, ProvisioningDevice, ProvisioningError +from videoipath_automation_tool.provisioning.inspect import InspectGateway + +from .conftest import make_inspect_app, matrox_layout + +if TYPE_CHECKING: + from videoipath_automation_tool.apps.inspect.model.actions import InspectApiLookupInspectDeviceResponse + from videoipath_automation_tool.provisioning import ProvisioningEngine + + from .conftest import FakeInspectServer, FakeInventory + + +def test_inventory_only_collector_node_is_added_before_configuration( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + inventory.seed("device1", label="device-a") + server.nodes["device1"] = {"_id": "device1", "modules": {}} + server.discoverable["device1"] = matrox_layout("device1") + original = server.lookup_inspect_device + + def lookup(device_id: str) -> InspectApiLookupInspectDeviceResponse: + if device_id not in server.device_forms: + raise InspectEntityNotFoundError(device_id, "device") + return original(device_id) + + monkeypatch.setattr(server, "lookup_inspect_device", lookup) + gateway = InspectGateway(make_inspect_app(server)) + assert not gateway.in_topology("device1") + assert gateway.read_peer_scope(DeviceTarget(device_id="device1")) is None + plan = engine.plan( + ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1"), + Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": {"device": {"tags": ["tag-a"]}}}}), + scope="topology", + ) + assert not plan.fully_resolved and not server.writes + result = plan.apply() + assert result.status == "succeeded" + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices", "update_topology"] + + +def test_membership_lookup_failure_is_not_treated_as_absence( + server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + server.nodes["device1"] = {"_id": "device1", "modules": {}} + + def fail(device_id: str) -> None: + raise ConnectionError("test transport failure") + + monkeypatch.setattr(server, "lookup_inspect_device", fail) + with pytest.raises(ProvisioningError, match="Inspect edit form"): + InspectGateway(make_inspect_app(server)).in_topology("device1") + assert not server.writes + + +def test_missing_collector_factory_labels_use_driver_graph_without_editable_names( + server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + layout = matrox_layout("device1") + server.install(layout) + labels: dict[str, str] = {} + expected: dict[str, str] = {} + for module in server.nodes["device1"]["modules"].values(): + for port in module["ports"].values(): + expected[port["pid"]] = port["label"] + info = port["vertexInfo"] + sides = [info["in"], info["out"]] if info["type"] == "double" else [info] + labels.update({side["id"]: port["label"] for side in sides}) + port["label"] = None + port["descriptor"] = {"label": "editable-name", "desc": ""} + calls: list[tuple[str, ...]] = [] + + def factory_labels(*vertex_ids: str) -> dict[str, str]: + calls.append(vertex_ids) + return labels + + monkeypatch.setattr(server, "get_ngraph_factory_labels", factory_labels, raising=False) + scope = InspectGateway(make_inspect_app(server)).read_scope(DeviceTarget(device_id="device1")) + assert {port.id: port.factory_label for port in scope.ports} == expected + assert all(port.label == "editable-name" for port in scope.ports) + assert all(vertex.factory_label == expected[vertex.port_id] for vertex in scope.vertices) + assert len(calls) == 1 and set(calls[0]) == set(labels) + assert not server.writes diff --git a/tests/blueprints/test_loader.py b/tests/provisioning/test_loader.py similarity index 71% rename from tests/blueprints/test_loader.py rename to tests/provisioning/test_loader.py index f5b4130..be4058f 100644 --- a/tests/blueprints/test_loader.py +++ b/tests/provisioning/test_loader.py @@ -6,14 +6,14 @@ import pytest -from videoipath_automation_tool.blueprints import Blueprint, BlueprintValidationError -from videoipath_automation_tool.blueprints.loader import MAX_DOCUMENT_BYTES, parse_yaml +from videoipath_automation_tool.provisioning import Blueprint, ProvisioningValidationError +from videoipath_automation_tool.provisioning.loader import MAX_DOCUMENT_BYTES, parse_yaml -MINIMAL = "schema_version: 1\ntopology:\n default: {}\n" +MINIMAL = "schema_version: 1\ndefaults:\n topology: {}\n" def _issue(text: str | bytes) -> str: - with pytest.raises(BlueprintValidationError) as info: + with pytest.raises(ProvisioningValidationError) as info: parse_yaml(text, source="bp.yml") return str(info.value) @@ -65,22 +65,22 @@ def test_size_depth_and_encoding_limits() -> None: def test_errors_carry_source_line_and_column() -> None: - text = "schema_version: 1\ntopology:\n default:\n vertex_processor:\n processor_type: x\n params:\n flag: 1\n extra: 2\n" - with pytest.raises(BlueprintValidationError) as info: + text = "schema_version: 1\ndefaults:\n topology:\n vertex_processor:\n processor_type: x\n params:\n flag: 1\n extra: 2\n" + with pytest.raises(ProvisioningValidationError) as info: Blueprint.from_yaml(text, source="bp.yml") issue = info.value.issues[0] assert (issue.source, issue.line, issue.column, issue.path) == ( "bp.yml", 8, 14, - "topology.default.vertex_processor.extra", + "defaults.topology.vertex_processor.extra", ) - assert str(issue).startswith("bp.yml:8:14\ntopology.default.vertex_processor.extra\n") + assert str(issue).startswith("bp.yml:8:14\ndefaults.topology.vertex_processor.extra\n") def test_load_from_file_and_bytes(tmp_path: Path) -> None: path = tmp_path / "bp.yml" path.write_text(MINIMAL, encoding="utf-8") - assert Blueprint.load(path).variant_names("topology") == ["default"] - assert Blueprint.from_yaml(MINIMAL.encode("utf-8")).topology is not None - assert Blueprint.from_yaml("\ufeff" + MINIMAL).topology is not None + assert Blueprint.load(path).variant_names() == [] + assert Blueprint.from_yaml(MINIMAL.encode("utf-8")).defaults.topology is not None + assert Blueprint.from_yaml("\ufeff" + MINIMAL).defaults.topology is not None diff --git a/tests/blueprints/test_naming.py b/tests/provisioning/test_naming.py similarity index 94% rename from tests/blueprints/test_naming.py rename to tests/provisioning/test_naming.py index 21aef80..4f1ffb0 100644 --- a/tests/blueprints/test_naming.py +++ b/tests/provisioning/test_naming.py @@ -5,16 +5,16 @@ import pytest from pydantic import ValidationError -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( DEFAULT_NAMING, - BlueprintValidationError, Field, Join, NameContext, NamingScheme, + ProvisioningValidationError, Text, ) -from videoipath_automation_tool.blueprints.naming import BlueprintNaming, render_name +from videoipath_automation_tool.provisioning.naming import BlueprintNaming, render_name def _context(position: str | None = None, index: int = 1, **attributes: object) -> NameContext: @@ -96,7 +96,7 @@ def test_static_validation(entry: str, expression: str, message: str) -> None: ], ) def test_render_errors(expression: object, code: str) -> None: - with pytest.raises(BlueprintValidationError) as info: + with pytest.raises(ProvisioningValidationError) as info: render_name("endpoint_label", expression, _context(nested={"a": 1})) assert info.value.codes == [code] @@ -107,7 +107,7 @@ def test_skip_missing_only_skips_absent_values() -> None: ) assert render_name("endpoint_label", join, _context()) == "device-a" failing = Join(parts=["{device.label}", Field("endpoint.media", mapping={"audio": "30"})], skip_missing=True) - with pytest.raises(BlueprintValidationError, match="no mapping"): + with pytest.raises(ProvisioningValidationError, match="no mapping"): render_name("endpoint_label", failing, _context()) diff --git a/tests/blueprints/test_processors.py b/tests/provisioning/test_processors.py similarity index 95% rename from tests/blueprints/test_processors.py rename to tests/provisioning/test_processors.py index af06d9e..8b45447 100644 --- a/tests/blueprints/test_processors.py +++ b/tests/provisioning/test_processors.py @@ -8,10 +8,8 @@ import pytest from pydantic import BaseModel, ConfigDict, ValidationError -from tests.blueprints.conftest import FakeInspectServer, make_inspect_app, matrox_layout -from videoipath_automation_tool.blueprints import ( - BlueprintCapabilityError, - BlueprintValidationError, +from tests.provisioning.conftest import FakeInspectServer, make_inspect_app, matrox_layout +from videoipath_automation_tool.provisioning import ( DeviceTarget, InterfaceBinding, MatroxConvertIPParams, @@ -21,12 +19,14 @@ ProcessorInputError, ProcessorRegistry, ProcessorResult, + ProvisioningCapabilityError, + ProvisioningValidationError, SourceFacts, TagDelta, TopologyNotReadyError, VertexProcessor, ) -from videoipath_automation_tool.blueprints.inspect import InspectGateway, build_context +from videoipath_automation_tool.provisioning.inspect import InspectGateway, build_context def _context( @@ -85,7 +85,7 @@ def test_registry_rules() -> None: registry.register("bad id", _Noop) with pytest.raises(TypeError, match="VertexProcessor subclass"): registry.register("example-org.x", object) # type: ignore[arg-type] - with pytest.raises(BlueprintValidationError, match="Available: example-org.noop, matrox.convertip.default"): + with pytest.raises(ProvisioningValidationError, match="Available: example-org.noop, matrox.convertip.default"): registry.get("missing") clone = registry.copy() @@ -95,7 +95,7 @@ def test_registry_rules() -> None: def test_params_validation_carries_paths() -> None: - with pytest.raises(BlueprintValidationError) as info: + with pytest.raises(ProvisioningValidationError) as info: ProcessorRegistry().validate_params("matrox.convertip.default", {"mode": "sideways"}, ("params",)) assert info.value.issues[0].path == "params.mode" @@ -268,7 +268,7 @@ def test_redundancy_validation() -> None: def test_control_configuration_is_an_explicit_capability_error() -> None: - with pytest.raises(BlueprintCapabilityError, match="control"): + with pytest.raises(ProvisioningCapabilityError, match="control"): _process(_context(matrox_layout("device1", "tx")), configure_control=True) diff --git a/tests/provisioning/test_readiness.py b/tests/provisioning/test_readiness.py new file mode 100644 index 0000000..3998f35 --- /dev/null +++ b/tests/provisioning/test_readiness.py @@ -0,0 +1,409 @@ +"""Independent rollout budgets, status gates, and delayed topology visibility, entirely offline.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest +from pydantic import ValidationError + +from tests.provisioning.conftest import FakeInspectServer, FakeInventory, build_layout, matrox_layout +from tests.provisioning.test_engine import _blueprint, _device, _existing +from videoipath_automation_tool.apps.inventory.model.device_status import DeviceStatus +from videoipath_automation_tool.apps.inventory.model.inventory_device import InventoryDevice +from videoipath_automation_tool.provisioning import ( + ApplyOptions, + Blueprint, + DeviceTarget, + InventoryNotReadyError, + ModuleTarget, + ProvisioningApplyError, + ProvisioningDevice, + ProvisioningEngine, + TopologyNotReadyError, +) + + +def _new_device() -> ProvisioningDevice: + return ProvisioningDevice(key="device-a", label="device-a", management_address="192.0.2.10") + + +def _on_sleep(engine: ProvisioningEngine, callback: Callable[[float], None]) -> None: + advance = engine._sleep + + def sleep(seconds: float) -> None: + advance(seconds) + callback(engine._clock()) + + engine._sleep = sleep + + +def test_inventory_gate_precedes_every_topology_write( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + inventory.status_sequence["device100"] = [None, False, True] + server.discoverable["device100"] = matrox_layout("device100", "tx") + read = inventory.refresh_device_status + + def status(device: InventoryDevice) -> InventoryDevice: + assert server.writes == [] + return read(device) + + monkeypatch.setattr(inventory, "refresh_device_status", status) + plan = engine.plan(_new_device(), _blueprint()) + assert inventory.status_reads == [] and engine._clock() == 0 + assert plan.phase("inventory_readiness").status == "deferred" + result = plan.apply() + assert result.status == "succeeded" and engine._clock() == 2 + assert inventory.status_reads == ["device100"] * 3 + assert result.phase("inventory_readiness").status == "completed" + assert result.phase("discovery").status == "completed" + assert [phase.name for phase in result.phases] == [ + "inventory", + "inventory_readiness", + "topology_sync", + "discovery", + "topology", + "module_tags", + "verification", + ] + + +@pytest.mark.parametrize("state", [None, False]) +def test_inventory_timeout_keeps_id_and_does_not_start_topology( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, state: bool | None +) -> None: + inventory.status_sequence["device100"] = [state] + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint(), options=ApplyOptions(inventory_ready_timeout=2.5)) + result = info.value.result + assert isinstance(info.value.__cause__, InventoryNotReadyError) + assert result.status == "partial" and result.inventory_id == "device100" + assert result.phase("inventory").status == "completed" + assert result.phase("inventory_readiness").status == "failed" + assert "device100" in result.phase("inventory_readiness").message + assert "2.5s" in result.phase("inventory_readiness").message + assert ("status unavailable" if state is None else "reachable=False") in result.phase("inventory_readiness").message + assert all(result.phase(name).status == "not_run" for name in ("topology_sync", "discovery", "topology")) + assert server.writes == [] and len(inventory.status_reads) == 3 and engine._clock() == 2.5 + + +def test_topology_gets_a_fresh_full_budget_after_slow_inventory( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.status_sequence["device100"] = [False] * 9 + [True] + + def discover(now: float) -> None: + if now == 68: + server.discoverable["device100"] = matrox_layout("device100", "tx") + + _on_sleep(engine, discover) + result = engine.apply(_new_device(), _blueprint()) + assert result.status == "succeeded" and engine._clock() == 68 + assert len(inventory.status_reads) == 10 + + +def test_reachability_bypass_is_explicit_and_reported( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.status_sequence["device100"] = [False] + server.discoverable["device100"] = matrox_layout("device100", "tx") + plan = engine.plan(_new_device(), _blueprint(), options=ApplyOptions(require_reachable=False)) + assert plan.phase("inventory_readiness").status == "skipped" + result = plan.apply() + assert result.status == "succeeded" and inventory.status_reads == [] + assert result.phase("inventory_readiness").status == "skipped" + assert "require_reachable=False" in result.phase("inventory_readiness").message + + +def test_accepted_add_and_sync_are_not_repeated_while_visibility_lags( + engine: ProvisioningEngine, server: FakeInspectServer +) -> None: + server.discoverable["device100"] = matrox_layout("device100", "tx") + server.add_visibility_polls = 4 + server.sync_completion_polls = 4 + result = engine.apply(_new_device(), _blueprint()) + assert result.status == "succeeded" and engine._clock() > 0 + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices", "update_topology"] + + +@pytest.mark.parametrize("stage", ["addition", "sync", "ports"]) +def test_topology_timeout_reports_the_stalled_stage( + engine: ProvisioningEngine, server: FakeInspectServer, stage: str +) -> None: + server.discoverable["device100"] = ( + build_layout("device100", {"device100.dev.0": []}) if stage == "ports" else matrox_layout("device100", "tx") + ) + server.add_visibility_polls = 100 if stage == "addition" else 0 + server.sync_completion_polls = 100 if stage == "sync" else 0 + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint(), options=ApplyOptions(topology_ready_timeout=2.5)) + result = info.value.result + assert isinstance(info.value.__cause__, TopologyNotReadyError) + failed_phase = "discovery" if stage == "ports" else "topology_sync" + assert result.phase(failed_phase).status == "failed" + assert "device100" in result.phase(failed_phase).message and "2.5s" in result.phase(failed_phase).message + assert result.phase("inventory_readiness").status == "completed" + assert result.phase("topology").status == "not_run" + assert result.inventory_id == "device100" and result.status == "partial" + assert [name for name, _ in server.writes].count("add_devices") == 1 + assert [name for name, _ in server.writes].count("sync_devices") == (0 if stage == "addition" else 1) + assert engine._clock() == 2.5 + + +def test_newly_discovered_ports_are_synchronized_before_configuration( + engine: ProvisioningEngine, server: FakeInspectServer +) -> None: + server.discoverable["device100"] = build_layout("device100", {"device100.dev.0": []}) + + def discover(now: float) -> None: + if now == 2: + server.discoverable["device100"] = matrox_layout("device100", "tx") + server.sync["device100"] = {"add": {"ports": 4}, "update": {}, "remove": {}} + + _on_sleep(engine, discover) + result = engine.apply(_new_device(), _blueprint()) + assert result.status == "succeeded" and engine._clock() == 2 + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices", "sync_devices", "update_topology"] + + +def test_retry_after_port_timeout_defers_existing_incomplete_topology( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + server.discoverable["device100"] = build_layout("device100", {"device100.dev.0": []}) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint(), options=ApplyOptions(topology_ready_timeout=2)) + retry = _new_device().model_copy(update={"inventory_id": info.value.result.inventory_id}) + _on_sleep(engine, lambda now: server.install(matrox_layout("device100", "tx"))) + plan = engine.plan(retry, _blueprint()) + assert not plan.fully_resolved and plan.phase("discovery").status == "deferred" + result = plan.apply() + assert result.status == "succeeded" and engine._clock() == 3 + assert inventory.writes == [("add", "device100")] + assert [name for name, _ in server.writes].count("add_devices") == 1 + + +@pytest.mark.parametrize("failure", ["transport", "value", "validation"]) +def test_inventory_status_errors_are_not_retried( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, failure: str +) -> None: + error: Exception = ConnectionError("offline") if failure == "transport" else ValueError("invalid response") + if failure == "validation": + with pytest.raises(ValidationError) as invalid: + DeviceStatus.model_validate({}) + error = invalid.value + inventory.status_sequence["device100"] = [error] + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint()) + assert info.value.__cause__ is error + assert info.value.result.phase("inventory_readiness").status == "failed" + assert len(inventory.status_reads) == 1 and engine._clock() == 0 and server.writes == [] + + +def test_slow_status_request_counts_against_inventory_deadline( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + read = inventory.refresh_device_status + + def slow_status(device: InventoryDevice) -> InventoryDevice: + engine._sleep(10) + return read(device) + + monkeypatch.setattr(inventory, "refresh_device_status", slow_status) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint()) + assert isinstance(info.value.__cause__, InventoryNotReadyError) + assert inventory.status_reads == ["device100"] and server.writes == [] + + +def test_slow_membership_read_cannot_start_a_write_after_deadline( + engine: ProvisioningEngine, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + read = server.get_device_detail + + def slow_detail(device_id: str) -> Any: + engine._sleep(60) + return read(device_id) + + monkeypatch.setattr(server, "get_device_detail", slow_detail) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint()) + assert isinstance(info.value.__cause__, TopologyNotReadyError) + assert info.value.result.phase("topology_sync").status == "failed" + assert server.calls == ["detail:device100"] and server.writes == [] + + +def test_slow_topology_data_read_does_not_start_another_read_after_deadline( + engine: ProvisioningEngine, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + server.discoverable["device100"] = matrox_layout("device100", "tx") + read = server.lookup_inspect_device + + def slow_form(device_id: str) -> Any: + # Membership now checks the edit form too; delay the discovery read after sync. + if any(name == "sync_devices" for name, _ in server.writes): + engine._sleep(60) + return read(device_id) + + monkeypatch.setattr(server, "lookup_inspect_device", slow_form) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint()) + assert isinstance(info.value.__cause__, TopologyNotReadyError) + assert info.value.result.phase("discovery").status == "failed" + assert not any(call.startswith("lookup_vertices") for call in server.calls) + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices"] + + +def test_late_sync_response_keeps_successful_operations_without_configuration_writes( + engine: ProvisioningEngine, server: FakeInspectServer, monkeypatch: pytest.MonkeyPatch +) -> None: + server.discoverable["device100"] = matrox_layout("device100", "tx") + sync = server.sync_devices + + def slow_sync(*args: Any, **kwargs: Any) -> Any: + response = sync(*args, **kwargs) + engine._sleep(60) + return response + + monkeypatch.setattr(server, "sync_devices", slow_sync) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint()) + result = info.value.result + phase = result.phase("topology_sync") + assert isinstance(info.value.__cause__, TopologyNotReadyError) + assert phase.status == "failed" and "synchronization accepted" in phase.message + assert [operation.action for operation in phase.operations] == ["add_to_topology", "sync"] + assert result.phase("discovery").status == "not_run" + assert result.inventory_id == "device100" and result.status == "partial" + assert [name for name, _ in server.writes] == ["add_devices", "sync_devices"] + + +def test_expiry_before_first_topology_poll_is_reported_in_topology_sync( + engine: ProvisioningEngine, server: FakeInspectServer +) -> None: + engine._sleep(1) + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(_new_device(), _blueprint(), options=ApplyOptions(topology_ready_timeout=1e-30)) + assert info.value.result.phase("inventory_readiness").status == "completed" + assert info.value.result.phase("topology_sync").status == "failed" + assert server.calls == [] and server.writes == [] + + +@pytest.mark.parametrize("stage", ["inventory", "topology"]) +def test_poll_interval_is_capped_to_the_remaining_stage_budget( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, stage: str +) -> None: + if stage == "inventory": + inventory.status_sequence["device100"] = [False] + options = ApplyOptions(inventory_ready_timeout=2.5, topology_ready_timeout=2.5, poll_interval=20) + with pytest.raises(ProvisioningApplyError): + engine.apply(_new_device(), _blueprint(), options=options) + assert engine._clock() == 2.5 and inventory.status_reads == ["device100"] + assert len(server.writes) == (0 if stage == "inventory" else 1) + + +def test_inventory_only_and_fully_resolved_topology_skip_the_gate( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.status_sequence["device100"] = [False] + result = engine.apply(_new_device(), _blueprint(), scope="inventory") + assert result.phase("inventory_readiness").status == "skipped" + _existing(inventory, server) + inventory.status_sequence["device1"] = [False] + result = engine.apply(_device(), _blueprint()) + assert result.phase("inventory_readiness").status == "skipped" + assert result.status == "succeeded" and inventory.status_reads == [] + + +@pytest.mark.parametrize("kind", ["virtual", "module"]) +def test_deferred_virtual_and_unbound_module_targets_skip_inventory_readiness( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, kind: str +) -> None: + device_id = "virtual.1" if kind == "virtual" else "device1" + module_id = f"{device_id}.dev.0" + server.install(build_layout(device_id, {module_id: []})) + target = ( + DeviceTarget(device_id=device_id) + if kind == "virtual" + else ModuleTarget(device_id=device_id, module_id=module_id) + ) + device = ProvisioningDevice(key="device-a", label="device-a", topology=target) + blueprint = Blueprint.from_dict( + {"schema_version": 1, "defaults": {"topology": {"ip_vertex_mapping": {"a": ["P1"]}}}} + ) + _on_sleep(engine, lambda now: server.install(matrox_layout(device_id, "tx"))) + plan = engine.plan(device, blueprint) + assert not plan.fully_resolved and plan.phase("inventory_readiness").status == "skipped" + result = plan.apply() + assert result.materialized and result.phase("inventory_readiness").status == "skipped" + assert inventory.status_reads == [] and engine._clock() == 1 + assert not any(name in {"add_devices", "sync_devices"} for name, _ in server.writes) + + +def test_topology_only_physical_target_checks_its_inventory_record( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.seed("device1") + inventory.status_sequence["device1"] = [False, True] + server.discoverable["device1"] = matrox_layout("device1", "tx") + device = ProvisioningDevice(key="device-a", label="device-a", topology=DeviceTarget(device_id="device1")) + result = engine.apply(device, _blueprint(), scope="topology") + assert result.status == "succeeded" and inventory.status_reads == ["device1", "device1"] + + +def test_deferred_module_checks_its_own_inventory_binding( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.seed("device1") + inventory.seed("device5", label="module-a", address="192.0.2.55") + inventory.status_sequence["device1"] = [False] + inventory.status_sequence["device5"] = [False, True] + server.install(build_layout("device1", {"device1.dev.0": []})) + device = ProvisioningDevice( + key="module-a", + label="module-a", + inventory_id="device5", + topology=ModuleTarget(device_id="device1", module_id="device1.dev.0"), + ) + blueprint = Blueprint.from_dict( + {"schema_version": 1, "defaults": {"topology": {"ip_vertex_mapping": {"a": ["P1"]}}}} + ) + _on_sleep(engine, lambda now: server.install(matrox_layout("device1", "tx")) if now == 2 else None) + result = engine.apply(device, blueprint, scope="topology") + assert result.materialized and result.phase("inventory_readiness").status == "completed" + assert inventory.status_reads == ["device5", "device5"] + assert inventory.writes == [] and not any(name in {"add_devices", "sync_devices"} for name, _ in server.writes) + + +def test_dry_run_does_not_poll_or_sleep( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.status_sequence["device100"] = [ConnectionError("must not read")] + result = engine.apply(_new_device(), _blueprint(), dry_run=True) + assert result.phase("inventory_readiness").status == "deferred" + assert result.phase("discovery").status == "deferred" + assert inventory.status_reads == [] and engine._clock() == 0 + assert inventory.writes == [] and server.writes == [] + + +def test_plans_capture_readiness_options(engine: ProvisioningEngine, inventory: FakeInventory) -> None: + inventory.status_sequence["device100"] = [False] + engine.configure(options=ApplyOptions(inventory_ready_timeout=2, topology_ready_timeout=3)) + plan = engine.plan(_new_device(), _blueprint()) + engine.configure(options=ApplyOptions(inventory_ready_timeout=100, require_reachable=False)) + with pytest.raises(ProvisioningApplyError) as info: + plan.apply() + assert isinstance(info.value.__cause__, InventoryNotReadyError) and engine._clock() == 2 + + +@pytest.mark.parametrize("field", ["inventory_ready_timeout", "topology_ready_timeout", "poll_interval"]) +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("-inf"), float("nan"), True]) +def test_readiness_times_must_be_positive_finite_numbers(field: str, value: Any) -> None: + with pytest.raises(ValidationError): + ApplyOptions(**{field: value}) + + +def test_old_timeout_option_is_rejected() -> None: + with pytest.raises(ValidationError, match="discovery_timeout"): + ApplyOptions(discovery_timeout=30) diff --git a/tests/blueprints/test_resolution.py b/tests/provisioning/test_resolution.py similarity index 55% rename from tests/blueprints/test_resolution.py rename to tests/provisioning/test_resolution.py index 216e0da..88c7c64 100644 --- a/tests/blueprints/test_resolution.py +++ b/tests/provisioning/test_resolution.py @@ -10,16 +10,16 @@ import pytest from pydantic import BaseModel, ConfigDict -from videoipath_automation_tool.blueprints import ( +from videoipath_automation_tool.provisioning import ( Blueprint, - BlueprintValidationError, InventorySettings, ProcessorRegistry, ProcessorResult, + ProvisioningValidationError, VertexProcessor, published_json_schema, ) -from videoipath_automation_tool.blueprints.resolution import resolve_blueprint +from videoipath_automation_tool.provisioning.resolution import resolve_blueprint NMOS = "com.nevion.NMOS_multidevice-0.1.0" @@ -27,18 +27,14 @@ def _document() -> dict[str, Any]: return { "schema_version": 1, - "inventory": { - "default": { + "defaults": { + "inventory": { "driver_id": NMOS, "generic_settings": {"enable_https": False, "http_auth": 0}, "custom_settings": {"port": 8080, "disable_rx_sdp_with_null": True}, "metadata": {"site": "site-a"}, }, - "secure": {"generic_settings": {"enable_https": True}, "custom_settings": {"port": 443}}, - "other-driver": {"driver_id": "com.nevion.NMOS-0.1.0", "custom_settings": {"always_enable_rtp": True}}, - }, - "topology": { - "default": { + "topology": { "device": {"icon_size": "medium", "tags": ["tag-a"]}, "vertex_processor": { "processor_type": "matrox.convertip.default", @@ -46,9 +42,19 @@ def _document() -> dict[str, Any]: }, "naming": {"endpoint_label": "{device.label}-{endpoint.index:02d}"}, }, - "receiver": {"vertex_processor": {"params": {"mode": "rx", "video_receiver_tags": []}}}, - "no-processor": {"vertex_processor": None, "device": {"tags": []}}, - "custom": {"vertex_processor": {"processor_type": "example-org.simple", "params": {"factory_label": "x"}}}, + }, + "variants": { + "secure": {"inventory": {"generic_settings": {"enable_https": True}, "custom_settings": {"port": 443}}}, + "other-driver": { + "inventory": {"driver_id": "com.nevion.NMOS-0.1.0", "custom_settings": {"always_enable_rtp": True}} + }, + "receiver": {"topology": {"vertex_processor": {"params": {"mode": "rx", "video_receiver_tags": []}}}}, + "no-processor": {"topology": {"vertex_processor": None, "device": {"tags": []}}}, + "custom": { + "topology": { + "vertex_processor": {"processor_type": "example-org.simple", "params": {"factory_label": "x"}} + } + }, }, } @@ -70,36 +76,36 @@ def _registry() -> ProcessorRegistry: def _resolve(blueprint: Blueprint, **kwargs: Any) -> Any: - options = {"scope": "all", "inventory_variant": "default", "topology_variant": "default", "registry": _registry()} + options = {"scope": "all", "registry": _registry()} options.update(kwargs) return resolve_blueprint(blueprint, **options) def test_named_variant_patches_default_and_records_provenance() -> None: - resolved = _resolve(Blueprint.from_dict(_document()), inventory_variant="secure", topology_variant="receiver") + resolved = _resolve(Blueprint.from_dict(_document()), variants=["secure", "receiver"]) assert resolved.inventory.custom_settings == {"port": 443, "disable_rx_sdp_with_null": True} assert resolved.inventory.config.generic_settings.managed() == {"enable_https": True, "http_auth": 0} assert resolved.inventory.provenance["custom_settings.port"] == "variant:secure" - assert resolved.inventory.provenance["custom_settings.disable_rx_sdp_with_null"] == "default" + assert resolved.inventory.provenance["custom_settings.disable_rx_sdp_with_null"] == "defaults" params = resolved.topology.params assert (params.mode, params.redundant_streams, params.video_receiver_tags) == ("rx", True, []) assert resolved.naming.entries() == {"endpoint_label": "{device.label}-{endpoint.index:02d}"} def test_null_disables_processor_and_empty_list_replaces() -> None: - resolved = _resolve(Blueprint.from_dict(_document()), topology_variant="no-processor") + resolved = _resolve(Blueprint.from_dict(_document()), variants=["no-processor"]) assert resolved.topology.processor_cls is None assert resolved.topology.config.device.managed() == {"icon_size": "medium", "tags": []} def test_processor_change_does_not_inherit_parameters() -> None: - resolved = _resolve(Blueprint.from_dict(_document()), topology_variant="custom") + resolved = _resolve(Blueprint.from_dict(_document()), variants=["custom"]) assert resolved.topology.processor_id == "example-org.simple" assert resolved.topology.params == _SimpleParams(factory_label="x") def test_driver_change_replaces_custom_settings_base() -> None: - resolved = _resolve(Blueprint.from_dict(_document()), inventory_variant="other-driver") + resolved = _resolve(Blueprint.from_dict(_document()), variants=["other-driver"]) assert resolved.inventory.driver_id == "com.nevion.NMOS-0.1.0" assert resolved.inventory.custom_settings == {"always_enable_rtp": True} assert resolved.inventory.config.metadata == {"site": "site-a"} @@ -114,17 +120,19 @@ def test_instance_overrides_apply_last() -> None: def test_unknown_variant_never_falls_back() -> None: - with pytest.raises(BlueprintValidationError, match="Available: default, receiver, no-processor, custom"): - _resolve(Blueprint.from_dict(_document()), topology_variant="reciever") + with pytest.raises( + ProvisioningValidationError, match="Available: secure, other-driver, receiver, no-processor, custom" + ): + _resolve(Blueprint.from_dict(_document()), variants=["reciever"]) def test_section_selection_rules() -> None: - topology_only = Blueprint.from_dict({"schema_version": 1, "topology": {"default": {}}}) + topology_only = Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": {}}}) assert _resolve(topology_only).skipped == {"inventory": "section absent from blueprint"} - with pytest.raises(BlueprintValidationError, match="no 'inventory' section"): + with pytest.raises(ProvisioningValidationError, match="no 'inventory' section"): _resolve(topology_only, scope="inventory") - with pytest.raises(BlueprintValidationError, match="absent 'inventory' section"): - _resolve(topology_only, inventory_variant="secure") + with pytest.raises(ProvisioningValidationError, match="Unknown variant"): + _resolve(topology_only, variants=["secure"]) assert _resolve(Blueprint.from_dict(_document()), scope="inventory").skipped == { "topology": "not in scope 'inventory'" } @@ -134,97 +142,104 @@ def test_source_document_is_not_mutated() -> None: document = _document() original = copy.deepcopy(document) blueprint = Blueprint.from_dict(document) - _resolve(blueprint, inventory_variant="secure", topology_variant="receiver") - _resolve(blueprint, inventory_variant="secure", topology_variant="receiver") + _resolve(blueprint, variants=["secure", "receiver"]) + _resolve(blueprint, variants=["secure", "receiver"]) assert document == original assert ( - blueprint.model_dump(exclude_unset=True, by_alias=True)["topology"]["receiver"] - == original["topology"]["receiver"] + blueprint.model_dump(exclude_unset=True, by_alias=True)["variants"]["receiver"] + == original["variants"]["receiver"] ) @pytest.mark.parametrize( ("patch", "message"), [ - ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"prot": 1}}}}, "Unknown custom setting"), - ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": "80"}}}}, "Expected an integer"), - ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"port": 0}}}}, "greater than or equal to 1"), - ({"inventory": {"default": {"driver_id": NMOS, "custom_settings": {"driver_id": NMOS}}}}, "driver_id"), - ({"inventory": {"default": {"driver_id": "example.unknown-1.0"}}}, "Unknown driver"), - ({"inventory": {"default": {"custom_settings": {}}}}, "needs a driver_id"), - ({"inventory": {"default": {"driver_id": NMOS, "generic_settings": {"enable_https": "false"}}}}, "boolean"), + ({"defaults": {"inventory": {"driver_id": NMOS, "custom_settings": {"prot": 1}}}}, "Unknown custom setting"), + ({"defaults": {"inventory": {"driver_id": NMOS, "custom_settings": {"port": "80"}}}}, "Expected an integer"), ( - {"inventory": {"default": {"driver_id": NMOS, "naming": {"endpoint_label": "x"}}}}, + {"defaults": {"inventory": {"driver_id": NMOS, "custom_settings": {"port": 0}}}}, + "greater than or equal to 1", + ), + ({"defaults": {"inventory": {"driver_id": NMOS, "custom_settings": {"driver_id": NMOS}}}}, "driver_id"), + ({"defaults": {"inventory": {"driver_id": "example.unknown-1.0"}}}, "Unknown driver"), + ({"defaults": {"inventory": {"custom_settings": {}}}}, "needs a driver_id"), + ({"defaults": {"inventory": {"driver_id": NMOS, "generic_settings": {"enable_https": "false"}}}}, "boolean"), + ( + {"defaults": {"inventory": {"driver_id": NMOS, "naming": {"endpoint_label": "x"}}}}, "not valid in the 'inventory'", ), ], ) def test_inventory_validation(patch: dict[str, Any], message: str) -> None: - with pytest.raises(BlueprintValidationError, match=message): + with pytest.raises(ProvisioningValidationError, match=message): Blueprint.from_dict({"schema_version": 1, **patch}).validate_full() @pytest.mark.parametrize( ("document", "message"), [ - ({"schema_version": 2, "topology": {"default": {}}}, "schema_version"), - ({"schema_version": 1}, "needs an 'inventory' or a 'topology'"), - ({"schema_version": 1, "topology": {"rx": {}}}, "must contain a 'default'"), - ({"schema_version": 1, "topology": {"default": {"unknown": 1}}}, "Unknown field"), - ({"schema_version": 1, "topology": {"default": {"device": {"label": None}}}}, "null is not supported"), + ({"schema_version": 2, "defaults": {"topology": {}}}, "schema_version"), + ({"schema_version": 1, "defaults": {}}, "needs an 'inventory' or a 'topology'"), + ({"schema_version": 1, "variants": {"rx": {"topology": {}}}}, "Field required"), + ({"schema_version": 1, "defaults": {"topology": {"unknown": 1}}}, "Unknown field"), + ({"schema_version": 1, "defaults": {"topology": {"device": {"label": None}}}}, "null is not supported"), ( - {"schema_version": 1, "topology": {"default": {"device": {"tags": {"add": ["a"], "remove": ["a"]}}}}}, + {"schema_version": 1, "defaults": {"topology": {"device": {"tags": {"add": ["a"], "remove": ["a"]}}}}}, "both add and remove", ), - ({"schema_version": 1, "topology": {"default": {"vertices": [{"fields": {}}]}}}, "exactly one of"), + ({"schema_version": 1, "defaults": {"topology": {"vertices": [{"fields": {}}]}}}, "exactly one of"), ( - {"schema_version": 1, "topology": {"default": {"naming": {"endpoint_label": "{device.nope}"}}}}, + {"schema_version": 1, "defaults": {"topology": {"naming": {"endpoint_label": "{device.nope}"}}}}, "unknown field", ), - ({"schema_version": 1, "topology": {"default": {"ip_vertex_mapping": {"a": []}}}}, "at least 1"), - ({"schema_version": "1", "topology": {"default": {}}}, "schema_version"), + ({"schema_version": 1, "defaults": {"topology": {"ip_vertex_mapping": {"a": []}}}}, "at least 1"), + ({"schema_version": "1", "defaults": {"topology": {}}}, "schema_version"), ], ) def test_document_validation(document: dict[str, Any], message: str) -> None: - with pytest.raises(BlueprintValidationError, match=message): + with pytest.raises(ProvisioningValidationError, match=message): Blueprint.from_dict(document) def test_full_validation_checks_unused_variants_and_reports_unavailable_processors() -> None: document = _document() - document["topology"]["broken"] = {"vertex_processor": {"params": {"redundant_streams": "false"}}} + document["variants"]["broken"] = {"topology": {"vertex_processor": {"params": {"redundant_streams": "false"}}}} blueprint = Blueprint.from_dict(document) - with pytest.raises(BlueprintValidationError) as info: + with pytest.raises(ProvisioningValidationError) as info: blueprint.validate_full(ProcessorRegistry()) codes = {issue.path: issue.code for issue in info.value.issues} - assert codes["topology.broken.vertex_processor.params.redundant_streams"] == "schema.bool_type" - assert codes["topology.custom.vertex_processor.processor_type"] == "processor.unavailable" + assert codes["variants.broken.topology.vertex_processor.params.redundant_streams"] == "schema.bool_type" + assert codes["variants.custom.topology.vertex_processor.processor_type"] == "processor.unavailable" blueprint.model_copy() # structural load did not need the plugin _resolve(blueprint, scope="inventory") # inventory-only use works without the plugin def test_planning_requires_selected_processor() -> None: blueprint = Blueprint.from_dict(_document()) - with pytest.raises(BlueprintValidationError, match="Unknown processor 'example-org.simple'"): - _resolve(blueprint, topology_variant="custom", registry=ProcessorRegistry()) + with pytest.raises(ProvisioningValidationError, match="Unknown processor 'example-org.simple'"): + _resolve(blueprint, variants=["custom"], registry=ProcessorRegistry()) def test_schema_export_matches_runtime_and_package() -> None: generated = Blueprint.json_schema(ProcessorRegistry()) assert generated == published_json_schema() - packaged = Path(__file__).parents[2] / "src/videoipath_automation_tool/blueprints/schemas/blueprint-v1.schema.json" + packaged = ( + Path(__file__).parents[2] / "src/videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json" + ) assert json.loads(packaged.read_text()) == generated - branches = generated["$defs"]["VertexProcessorSpec"]["allOf"] + branches = generated["$defs"]["VertexProcessorSpecTemplate"]["allOf"] assert [b["if"]["properties"]["processor_type"]["const"] for b in branches] == ["matrox.convertip.default"] custom = Blueprint.json_schema(_registry()) - assert len(custom["$defs"]["VertexProcessorSpec"]["allOf"]) == 2 - assert "allOf" not in Blueprint.json_schema()["$defs"]["VertexProcessorSpec"] + assert len(custom["$defs"]["VertexProcessorSpecTemplate"]["allOf"]) == 2 + assert "allOf" not in Blueprint.json_schema()["$defs"]["VertexProcessorSpecTemplate"] def test_schema_can_restrict_processor_types_to_registered_ids() -> None: - open_spec = Blueprint.json_schema(_registry())["$defs"]["VertexProcessorSpec"] + open_spec = Blueprint.json_schema(_registry())["$defs"]["VertexProcessorSpecTemplate"] assert "enum" not in open_spec["properties"]["processor_type"] - restricted = Blueprint.json_schema(_registry(), restrict_processor_types=True)["$defs"]["VertexProcessorSpec"] + restricted = Blueprint.json_schema(_registry(), restrict_processor_types=True)["$defs"][ + "VertexProcessorSpecTemplate" + ] processor_type = restricted["properties"]["processor_type"] assert processor_type["enum"] == ["example-org.simple", "matrox.convertip.default"] assert processor_type["type"] == "string" and "anyOf" not in processor_type @@ -235,7 +250,7 @@ def test_schema_can_restrict_processor_types_to_registered_ids() -> None: def test_documented_example_blueprint_is_valid() -> None: - example = Path(__file__).parents[2] / "docs/examples/07_blueprints/matrox-convertip.yml" + example = Path(__file__).parents[2] / "docs/examples/07_provisioning/matrox-convertip.yml" blueprint = Blueprint.load(example) blueprint.validate_full(ProcessorRegistry()) - assert blueprint.variant_names("topology") == ["default", "receiver", "four-split"] + assert blueprint.variant_names() == ["receiver", "four-split"] diff --git a/tests/provisioning/test_v2.py b/tests/provisioning/test_v2.py new file mode 100644 index 0000000..1b6c039 --- /dev/null +++ b/tests/provisioning/test_v2.py @@ -0,0 +1,379 @@ +"""Typed inputs and ordered overlays, exercised through resolution and the real engine.""" + +from __future__ import annotations + +import copy +from types import SimpleNamespace +from typing import Any + +import pytest +from pydantic import BaseModel, ConfigDict, JsonValue + +from tests.provisioning.conftest import NMOS, FakeInspectServer, FakeInventory, matrox_layout +from videoipath_automation_tool.provisioning import ( + Blueprint, + InputDefinition, + InventorySettings, + NamingScheme, + ProcessorRegistry, + ProcessorResult, + ProvisioningDevice, + ProvisioningEngine, + ProvisioningValidationError, + VertexProcessor, +) +from videoipath_automation_tool.provisioning.naming import NameContext, render_name +from videoipath_automation_tool.provisioning.resolution import resolve_blueprint + + +def _document() -> dict[str, Any]: + return { + "schema_version": 1, + "inputs": { + "port": {"type": "integer", "default": 8080}, + "enabled": {"type": "boolean", "default": False}, + "tags": {"type": "array", "items": {"type": "string"}, "default": []}, + "site": { + "type": "object", + "properties": {"code": {"type": "string"}}, + "required": ["code"], + "default": {"code": "site-a"}, + }, + }, + "defaults": { + "inventory": { + "driver_id": NMOS, + "custom_settings": {"port": {"$input": "port"}}, + "generic_settings": {"enable_https": {"$input": "enabled"}}, + "metadata": {"site": "site-a"}, + }, + "topology": { + "vertex_processor": { + "processor_type": "matrox.convertip.default", + "params": {"video_sender_tags": {"$input": "tags"}}, + }, + "naming": {"endpoint_label": "{device.label}-{inputs.site.code}-{endpoint.media}-{endpoint.index:02d}"}, + }, + }, + "variants": { + "secure": {"inventory": {"custom_settings": {"port": 443}}}, + "alternate": {"inventory": {"custom_settings": {"port": 9000}}}, + "receiver": {"topology": {"vertex_processor": {"params": {"mode": "rx"}}}}, + }, + } + + +def _blueprint(**updates: Any) -> Blueprint: + document = _document() + document.update(updates) + return Blueprint.from_dict(document) + + +def _resolve(blueprint: Blueprint, **kwargs: Any) -> Any: + return resolve_blueprint(blueprint, registry=ProcessorRegistry(), **kwargs) + + +def test_inputs_preserve_types_and_provenance_and_overrides_apply_last() -> None: + resolved = _resolve(_blueprint(), inputs={"port": 9090}) + assert resolved.inventory.custom_settings == {"port": 9090} + assert resolved.inventory.config.generic_settings.enable_https is False + assert resolved.topology.params.video_sender_tags == [] + assert resolved.inventory.provenance["custom_settings.port"] == "defaults|input:port" + assert resolved.topology.provenance["vertex_processor.params.video_sender_tags"] == "defaults|input:tags" + overridden = _resolve( + _blueprint(), inputs={"port": 9090}, overrides=InventorySettings(custom_settings={"port": 10000}) + ) + assert overridden.inventory.custom_settings == {"port": 10000} + assert overridden.inventory.provenance["custom_settings.port"] == "instance" + + +@pytest.mark.parametrize( + ("inputs", "code"), + [ + ({"port": True}, "input.type"), + ({"port": "8080"}, "input.type"), + ({"port": 8080.0}, "input.type"), + ({"enabled": 0}, "input.type"), + ({"tags": [False]}, "input.type"), + ({"site": {}}, "input.missing"), + ({"site": {"code": "site-a", "unknown": 1}}, "input.unknown_property"), + ({"other": 1}, "input.unknown"), + ], +) +def test_bad_inputs_fail_before_app_access(inputs: dict[str, JsonValue], code: str) -> None: + engine = ProvisioningEngine(SimpleNamespace()) + device = ProvisioningDevice(key="device-a", label="device-a") + with pytest.raises(ProvisioningValidationError) as error: + engine.plan(device, _blueprint(), inputs=inputs) + assert code in {issue.code for issue in error.value.issues} + + +@pytest.mark.parametrize( + "definition", + [ + {"type": "integer", "default": True}, + {"type": "boolean", "default": "false"}, + {"type": "integer", "enum": [False]}, + {"type": "integer", "enum": [1], "default": 2}, + {"type": "array", "items": {"type": "string"}, "default": [1]}, + {"type": "object", "default": {"undeclared": 1}}, + {"type": "object", "required": ["missing"]}, + {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a", "a"]}, + {"type": "string", "items": {"type": "string"}}, + {"type": "string", "properties": {}}, + ], +) +def test_invalid_input_declarations_fail_on_load(definition: dict[str, Any]) -> None: + with pytest.raises(ProvisioningValidationError): + Blueprint.from_dict({"schema_version": 1, "inputs": {"bad": definition}, "defaults": {"topology": {}}}) + + +def test_all_input_types_and_open_objects() -> None: + definitions = { + "text": {"type": "string", "default": ""}, + "integer": {"type": "integer", "default": 0}, + "number": {"type": "number", "default": 1.5}, + "boolean": {"type": "boolean", "default": False}, + "array": {"type": "array", "default": [None, {"a": 1}]}, + "object": {"type": "object", "additionalProperties": True, "default": {"a": [None]}}, + "null_value": {"type": "null", "default": None}, + } + blueprint = Blueprint.from_dict({"schema_version": 1, "inputs": definitions, "defaults": {"topology": {}}}) + assert _resolve(blueprint).inputs == {name: spec["default"] for name, spec in definitions.items()} + assert InputDefinition(type="number", default=2).default == 2 + + +def test_requiredness_uses_effective_scope_and_overlays() -> None: + document = _document() + document["inputs"]["port"].pop("default") + document["defaults"]["inventory"]["metadata"] = {} + blueprint = Blueprint.from_dict(document) + with pytest.raises(ProvisioningValidationError, match="Required input is missing"): + _resolve(blueprint) + assert _resolve(blueprint, scope="topology").inventory is None + assert _resolve(blueprint, variants=["secure"]).inventory.custom_settings == {"port": 443} + blueprint.validate_full(ProcessorRegistry(), variants=["secure"]) + with pytest.raises(ProvisioningValidationError, match="Required input is missing"): + blueprint.validate_full(ProcessorRegistry()) + + +def test_overlay_order_and_cross_section_selection() -> None: + blueprint = _blueprint() + first = _resolve(blueprint, variants=["secure", "alternate", "receiver"]) + second = _resolve(blueprint, variants=["alternate", "secure"]) + assert first.inventory.custom_settings == {"port": 9000} + assert first.topology.params.mode == "rx" + assert first.variants == ("secure", "alternate", "receiver") + assert second.inventory.custom_settings == {"port": 443} + assert first.inventory.provenance["custom_settings.port"] == "variant:alternate" + + +@pytest.mark.parametrize("variants", [["secure", "secure"], ["missing"]]) +def test_invalid_variant_selection(variants: list[str]) -> None: + with pytest.raises(ProvisioningValidationError): + _resolve(_blueprint(), variants=variants) + + +@pytest.mark.parametrize("variants", ["secure", {"secure"}, [1]]) +def test_variant_argument_must_be_a_sequence_of_names(variants: Any) -> None: + with pytest.raises(TypeError, match="sequence of names"): + _resolve(_blueprint(), variants=variants) + + +def test_overlay_can_add_a_section() -> None: + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "defaults": {"topology": {}}, + "variants": {"onboard": {"inventory": {"driver_id": NMOS}}}, + } + ) + assert _resolve(blueprint, variants=["onboard"], scope="inventory").inventory.driver_id == NMOS + + +class _PayloadParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + payload: JsonValue + + +class _PayloadProcessor(VertexProcessor[_PayloadParams]): + params_model = _PayloadParams + + def process(self, context: Any, params: _PayloadParams) -> ProcessorResult: + return ProcessorResult() + + +def _payload_blueprint(payload: Any, inputs: dict[str, Any] | None = None) -> Blueprint: + return Blueprint.from_dict( + { + "schema_version": 1, + "inputs": inputs or {}, + "defaults": { + "topology": { + "vertex_processor": {"processor_type": "example-org.payload", "params": {"payload": payload}} + } + }, + } + ) + + +def _payload(blueprint: Blueprint, **kwargs: Any) -> Any: + registry = ProcessorRegistry({"example-org.payload": _PayloadProcessor}) + return resolve_blueprint(blueprint, registry=registry, **kwargs).topology.params.payload + + +def test_nested_references_and_literal_escapes_are_not_reinterpreted() -> None: + blueprint = _payload_blueprint( + [{"$input": "facts"}, {"$literal": {"$input": "not-declared"}}], + {"facts": {"type": "object", "additionalProperties": True}}, + ) + value = {"$input": "also-not-declared", "nested": [False, 0, ""]} + assert _payload(blueprint, inputs={"facts": value}) == [value, {"$input": "not-declared"}] + + +def test_whole_object_reference_is_atomic_during_merge() -> None: + blueprint = _payload_blueprint({"a": 1, "b": 2}, {"replacement": {"type": "object", "additionalProperties": True}}) + document = blueprint.model_dump(by_alias=True, exclude_unset=True) + document["variants"] = { + "replace": {"topology": {"vertex_processor": {"params": {"payload": {"$input": "replacement"}}}}} + } + assert _payload(Blueprint.from_dict(document), variants=["replace"], inputs={"replacement": {"c": 3}}) == {"c": 3} + + +@pytest.mark.parametrize( + "payload", [{"$input": "unknown"}, {"$input": "known", "extra": 1}, {"$literal": 1, "$input": "known"}] +) +def test_unknown_or_malformed_references_fail_on_load(payload: Any) -> None: + with pytest.raises(ProvisioningValidationError): + _payload_blueprint(payload, {"known": {"type": "string"}}) + + +def test_static_ids_and_unknown_fields_cannot_be_parameterized() -> None: + document = _blueprint().model_dump(by_alias=True, exclude_unset=True) + document["defaults"]["inventory"]["driver_id"] = {"$input": "port"} + with pytest.raises(ProvisioningValidationError): + Blueprint.from_dict(document) + document["defaults"]["inventory"]["driver_id"] = NMOS + document["defaults"]["topology"]["vertex_processor"] = {"$input": "port"} + with pytest.raises(ProvisioningValidationError): + Blueprint.from_dict(document) + document["defaults"]["topology"]["vertex_processor"] = {"processor_type": "matrox.convertip.default"} + document["defaults"]["inventory"]["typo"] = {"$input": "port"} + with pytest.raises(ProvisioningValidationError, match="Unknown field"): + Blueprint.from_dict(document) + + +def test_reference_error_and_driver_validation_keep_yaml_origin() -> None: + text = f"""schema_version: 1 +inputs: + port: {{type: integer, default: 0}} +defaults: + inventory: + driver_id: {NMOS} + custom_settings: + port: {{$input: port}} +""" + blueprint = Blueprint.from_yaml(text, source="device.yml") + with pytest.raises(ProvisioningValidationError) as error: + _resolve(blueprint) + issue = error.value.issues[0] + assert (issue.path, issue.source, issue.line) == ("defaults.inventory.custom_settings.port", "device.yml", 8) + with pytest.raises(ProvisioningValidationError) as error: + Blueprint.from_yaml(text.replace("$input: port", "$input: unknown"), source="device.yml") + assert error.value.issues[0].line == 8 + + +def test_naming_inputs_include_object_leaves_and_keep_literal_text() -> None: + naming = NamingScheme( + endpoint_label={"join": [{"text": "{inputs.literal}"}, {"field": "inputs.site.code"}], "separator": "-"} + ) + assert ( + render_name("endpoint_label", naming.endpoint_label, NameContext(inputs={"site": {"code": "site-a"}})) + == "{inputs.literal}-site-a" + ) + blueprint = _blueprint() + document = blueprint.model_dump(by_alias=True, exclude_unset=True) + document["defaults"]["topology"]["naming"] = {"endpoint_label": "{inputs.site.unknown}"} + with pytest.raises(ProvisioningValidationError, match="Unknown property"): + Blueprint.from_dict(document) + + +def test_engine_uses_and_captures_inputs_for_all_naming_targets( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + inventory.seed("device1", label="device-a", custom={"port": 8080}) + server.install(matrox_layout("device1", "tx")) + document = _blueprint().model_dump(by_alias=True, exclude_unset=True) + document["defaults"]["inventory"]["naming"] = {"inventory_label": "{inputs.site.code}-{device.label}"} + document["defaults"]["topology"]["naming"]["device_label"] = "{inputs.site.code}-{device.label}" + blueprint = Blueprint.from_dict(document) + original = copy.deepcopy(document) + inputs: dict[str, Any] = {"site": {"code": "site-b"}, "tags": ["video-tag-a"]} + variants: list[str] = [] + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") + plan = engine.plan(device, blueprint, inputs=inputs, variants=variants) + inputs["site"]["code"] = "changed" + inputs["tags"].append("changed") + variants.append("receiver") + result = plan.apply() + assert result.ok and plan.variants == () + assert inventory.devices["device1"].configuration.config.desc.label == "site-b-device-a" + assert server.device_forms["device1"]["descriptor"]["label"] == "site-b-device-a" + changes = [change for operation in result.phase("topology").operations for change in operation.changes] + assert any(change.after == "device-a-site-b-video-01" for change in changes) + assert blueprint.model_dump(by_alias=True, exclude_unset=True) == original + assert ( + engine.apply(device, blueprint, inputs={"site": {"code": "site-b"}, "tags": ["video-tag-a"]}).status + == "no_change" + ) + + +def test_combined_variants_are_validated_and_captured_before_app_access() -> None: + blueprint = _blueprint() + engine = ProvisioningEngine(SimpleNamespace()) + engine.validate(blueprint, variants=["secure", "receiver"]) + device = ProvisioningDevice(key="device-a", label="device-a") + with pytest.raises(ProvisioningValidationError): + engine.plan(device, blueprint, inputs={"port": "secret-value"}) + with pytest.raises(ProvisioningValidationError) as error: + engine.validate(blueprint, inputs={"port": "secret-value"}) + assert "secret-value" not in str(error.value) + + +@pytest.mark.parametrize("naming", ["{inputs.unknown}", "{inputs.site}", "{inputs.site.code}"]) +def test_call_time_naming_input_validation_happens_before_reads(naming: str) -> None: + engine = ProvisioningEngine(SimpleNamespace()) + document = _blueprint().model_dump(by_alias=True, exclude_unset=True) + document["inputs"]["site"].pop("default") + document["defaults"].pop("topology") + device = ProvisioningDevice(key="device-a", label="device-a") + values = {"site": {"code": "site-a"}} if naming == "{inputs.site}" else None + with pytest.raises(ProvisioningValidationError): + engine.plan(device, Blueprint.from_dict(document), inputs=values, naming=NamingScheme(inventory_label=naming)) + + +def test_unused_naming_inputs_do_not_affect_inventory_only_plans(inventory: FakeInventory) -> None: + inventory.seed("device1", label="device-a", custom={"port": 8080}) + engine = ProvisioningEngine( + SimpleNamespace(inventory=inventory), naming=NamingScheme(endpoint_label="{inputs.tags}") + ) + device = ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1") + assert engine.plan(device, _blueprint(), scope="inventory").fully_resolved + + +@pytest.mark.parametrize("version", [True, 2.0, "2"]) +def test_schema_version_is_strictly_an_integer(version: Any) -> None: + with pytest.raises(ProvisioningValidationError): + Blueprint.from_dict({"schema_version": version, "defaults": {"topology": {}}}) + + +def test_full_validation_reports_errors_in_both_sections() -> None: + document = _blueprint().model_dump(by_alias=True, exclude_unset=True) + document["defaults"]["inventory"]["custom_settings"]["port"] = 0 + document["defaults"]["topology"]["vertex_processor"]["params"]["mode"] = "invalid" + with pytest.raises(ProvisioningValidationError) as error: + Blueprint.from_dict(document).validate_full(ProcessorRegistry(), variants=[]) + assert {issue.path for issue in error.value.issues} == { + "defaults.inventory.custom_settings.port", + "defaults.topology.vertex_processor.params.mode", + } diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py new file mode 100644 index 0000000..51b0931 --- /dev/null +++ b/tests/test_test_runner.py @@ -0,0 +1,75 @@ +"""Check real pytest collection without contacting a server or running E2E fixtures.""" + +from __future__ import annotations + +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest + +from vipat_cli_scripts import test_runner as runner + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.parametrize( + ("selection", "expected"), + [ + ([], {"test_provision", "test_connection", "test_other"}), + (["tests/e2e/provisioning"], {"test_provision", "test_connection"}), + (["tests/e2e/provisioning/test_scenario.py"], {"test_provision", "test_connection"}), + (["tests/e2e/provisioning/test_scenario.py::test_provision"], {"test_provision"}), + (["-k", "connection"], {"test_connection"}), + ], + ids=["default", "directory", "file", "node", "options-only"], +) +def test_e2e_entry_point_selects_only_requested_tests(tmp_path: Path, selection: list[str], expected: set[str]) -> None: + (tmp_path / "pytest.ini").write_text("[pytest]\nmarkers = e2e: live test marker\n", encoding="utf-8") + scenarios = tmp_path / "tests" / "e2e" / "provisioning" + scenarios.mkdir(parents=True) + prefix = "import pytest\npytestmark = pytest.mark.e2e\n" + (scenarios / "test_scenario.py").write_text( + prefix + "def test_provision(): pass\ndef test_connection(): pass\n", encoding="utf-8" + ) + (scenarios.parent / "test_other.py").write_text(prefix + "def test_other(): pass\n", encoding="utf-8") + (tmp_path / "tests" / "test_unit.py").write_text("def test_unit(): pass\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + "-c", + "from vipat_cli_scripts.test_runner import run_e2e; run_e2e()", + *selection, + "--collect-only", + "-q", + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line.rsplit("::", 1)[-1] for line in result.stdout.splitlines() if "::" in line} + assert collected == expected + + +@pytest.mark.parametrize("unit_status", [0, 1]) +def test_combined_entry_point_retains_default_suite_and_stops_on_unit_failure( + monkeypatch: pytest.MonkeyPatch, unit_status: int +) -> None: + calls: list[list[str]] = [] + prepared: list[bool] = [] + + def run(args: list[str], *, extra: list[str]) -> int: + assert extra == [] + calls.append(args) + return unit_status if args is runner._UNIT_ARGS else 0 + + monkeypatch.setattr(runner, "_run", run) + monkeypatch.setattr(runner, "prepare_e2e_env", lambda: prepared.append(True)) + with pytest.raises(SystemExit) as caught: + runner.run() + assert caught.value.code == unit_status + assert calls == ([runner._UNIT_ARGS] if unit_status else [runner._UNIT_ARGS, runner._E2E_ARGS]) + assert prepared == ([] if unit_status else [True]) From 068ada3c187cfd4ad835c73952ee1a0f91cb5a82 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:30:56 +0200 Subject: [PATCH 5/7] change connections terminology to edges --- docs/architecture/provisioning/README.md | 4 +- docs/architecture/provisioning/concepts.md | 4 +- .../decisions/008-staged-readiness.md | 2 +- ...l-connections.md => 009-external-edges.md} | 16 +- .../provisioning/decisions/README.md | 2 +- docs/architecture/provisioning/reference.md | 69 ++++--- ...al_connections.py => 03_external_edges.py} | 16 +- ...nal-connections.yml => external-edges.yml} | 0 docs/examples/README.md | 2 +- docs/getting-started-guide/05_Provisioning.md | 10 +- .../provisioning/__init__.py | 8 +- .../provisioning/{connections.py => edges.py} | 60 +++--- .../provisioning/engine.py | 92 ++++----- .../provisioning/inspect.py | 52 +++-- .../provisioning/models.py | 29 +-- .../{test_connections.py => test_edges.py} | 38 ++-- tests/e2e/provisioning/test_recovery.py | 14 +- ...nnection_inputs.py => test_edge_inputs.py} | 8 +- .../{test_connections.py => test_edges.py} | 181 ++++++++++++------ tests/test_test_runner.py | 10 +- 20 files changed, 339 insertions(+), 278 deletions(-) rename docs/architecture/provisioning/decisions/{009-external-connections.md => 009-external-edges.md} (75%) rename docs/examples/07_provisioning/{03_external_connections.py => 03_external_edges.py} (75%) rename docs/examples/07_provisioning/{external-connections.yml => external-edges.yml} (100%) rename src/videoipath_automation_tool/provisioning/{connections.py => edges.py} (86%) rename tests/e2e/provisioning/{test_connections.py => test_edges.py} (84%) rename tests/provisioning/{test_connection_inputs.py => test_edge_inputs.py} (94%) rename tests/provisioning/{test_connections.py => test_edges.py} (68%) diff --git a/docs/architecture/provisioning/README.md b/docs/architecture/provisioning/README.md index a6abfd4..b1a9f80 100644 --- a/docs/architecture/provisioning/README.md +++ b/docs/architecture/provisioning/README.md @@ -50,7 +50,7 @@ offline/CI test run. The scenarios cover YAML inputs and variants, preview and dry run, onboarding, Inventory/topology scope isolation, managed-field preservation, endpoint naming, -module-local tags, directed connections, deferred peers, rediscovery and sync +module-local tags, directed edges, deferred peers, rediscovery and sync policies, stale plans, and recovery from an Inventory readiness timeout. Each scenario owns its devices and checks persisted values through fresh server reads. Mutation spies forward calls to the real SDK and verify that dry runs and @@ -63,7 +63,7 @@ resources, including resources from other E2E scenarios, before creating new ones. Run live sessions serially against an instance. Discovery polling is bounded; transport errors and unexpected failures are not retried. -External connection behavior is recorded in [ADR-009](./decisions/009-external-connections.md). +External edge behavior is recorded in [ADR-009](./decisions/009-external-edges.md). ## Decision log diff --git a/docs/architecture/provisioning/concepts.md b/docs/architecture/provisioning/concepts.md index 6e75e1e..e48a938 100644 --- a/docs/architecture/provisioning/concepts.md +++ b/docs/architecture/provisioning/concepts.md @@ -17,7 +17,7 @@ caller can apply that description to many real devices. The caller supplies the instance facts (`ProvisioningDevice`): source-system id, label, addresses, credentials, an optional Inventory id, and an optional -Inspect target, and concrete external connections. The blueprint supplies the type: driver settings, topology +Inspect target, and concrete external edges. The blueprint supplies the type: driver settings, topology appearance, vertex interpretation, port mapping, and naming. The engine does not know NetBox or any other source system. Translation into `ProvisioningDevice` stays outside the package. @@ -75,7 +75,7 @@ keys, or custom tags. Limits are 1 MiB, 32 levels, and 100,000 nodes. | `naming.py` | `Text` / `Field` / `Join` expressions and layering. | | `processors/` | `VertexProcessor`, detached records, per-engine `ProcessorRegistry`, Matrox built-in. | | `inventory.py` | `InventoryGateway`: one Inventory record, managed-field diff, create/update. | -| `connections.py` | Pure generic port resolution, directional edge selection and managed-field diffs. | +| `edges.py` | Pure generic port resolution, directional edge selection and managed-field diffs. | | `inspect.py` | `InspectGateway`: fresh scoped read, processor run, topology commit, module tags. | | `engine.py` | Configuration, `plan()`, captured inputs, phased `apply()`. | | `errors.py` | `ProvisioningError` hierarchy. Validation issues carry a path, not raw values. | diff --git a/docs/architecture/provisioning/decisions/008-staged-readiness.md b/docs/architecture/provisioning/decisions/008-staged-readiness.md index 14b3f56..5e002ba 100644 --- a/docs/architecture/provisioning/decisions/008-staged-readiness.md +++ b/docs/architecture/provisioning/decisions/008-staged-readiness.md @@ -56,7 +56,7 @@ Unknown write outcomes are never automatically retried. Planning converts local `TopologyNotReadyError` into deferred topology work, including when an earlier rollout already added the device. Planning and dry runs never sleep, and dry runs report deferred readiness without rollout actions. -Missing peer connections continue to require a new plan. +Missing peer edges continue to require a new plan. ## Consequences diff --git a/docs/architecture/provisioning/decisions/009-external-connections.md b/docs/architecture/provisioning/decisions/009-external-edges.md similarity index 75% rename from docs/architecture/provisioning/decisions/009-external-connections.md rename to docs/architecture/provisioning/decisions/009-external-edges.md index a6b4058..179a5a1 100644 --- a/docs/architecture/provisioning/decisions/009-external-connections.md +++ b/docs/architecture/provisioning/decisions/009-external-edges.md @@ -1,4 +1,4 @@ -# ADR-009: Concrete external connections belong to the device instance +# ADR-009: Concrete external edges belong to the device instance > Status: **Accepted** > @@ -6,7 +6,7 @@ ## Context -Provisioning a device includes configuring its external connections. Reusable +Provisioning a device includes configuring its external edges. Reusable device templates must remain independent of instance-specific cabling and the system supplying it. IP interfaces and directed media ports need the same flow. @@ -17,8 +17,8 @@ in the blueprint document. Existing overlay and typed-input semantics apply. The `ip_vertex_mapping` and processor `interfaces` retain their existing behavior; generic mappings are exposed as `port_bindings`. -`ProvisioningDevice.connections` declares each concrete connection through -`ProvisioningConnection`, `PeerEndpoint` and `EdgePatch`. The local port is a +`ProvisioningDevice.edges` declares each concrete edge through +`ProvisioningEdge`, `PeerEndpoint` and `EdgePatch`. The local port is a mapping key or direct selector. Peers use exact VideoIPath device/module ids and port selectors. Source-system adapters remain outside provisioning. @@ -30,10 +30,10 @@ in explicitly managed fields. Other edges are preserved; empty input never prune The Inspect gateway reads peers and edge forms freshly, checks captured baselines and staged-edit overlap, and commits device, vertex and edge intents together in the existing topology transaction. Pure resolution/diff logic lives in -`connections.py`; processors receive no gateway or external write scope. +`edges.py`; processors receive no gateway or external write scope. -Missing peer discovery is reported per connection as `deferred`. Remaining local -work and resolved connections can run. The result is `partial` with +Missing peer discovery is reported per edge as `deferred`. Remaining local +work and resolved edges can run. The result is `partial` with `replan_required=True`, even if no write was necessary. A new plan is needed when peers appear; applying an old plan does not adopt new peer topology. Invalid selectors, ambiguous directions and failed reads are errors, not pending work. @@ -41,7 +41,7 @@ selectors, ambiguous directions and failed reads are errors, not pending work. ## Consequences - Blueprint and instance facts fully describe the requested configuration. -- One connection can produce one directed edge or both directions. +- One `ProvisioningEdge` declaration can produce one directed VideoIPath edge or both directions. - Missing peers do not prevent otherwise valid local onboarding. - Plans and results expose bindings, concrete edge ids and unresolved reasons. - No peer provisioning, implicit synchronization, pruning or rollback is added. diff --git a/docs/architecture/provisioning/decisions/README.md b/docs/architecture/provisioning/decisions/README.md index cbdc6ed..d9d68fd 100644 --- a/docs/architecture/provisioning/decisions/README.md +++ b/docs/architecture/provisioning/decisions/README.md @@ -16,4 +16,4 @@ than rewriting history. | [006](./006-variant-merge.md) | Variants are a fixed patch over `default` | Superseded by ADR-007 | | [007](./007-typed-inputs.md) | Typed inputs and ordered overlays | Accepted | | [008](./008-staged-readiness.md) | Independent Inventory and topology readiness deadlines | Accepted | -| [009](./009-external-connections.md) | Concrete external connections belong to the device instance | Accepted | +| [009](./009-external-edges.md) | Concrete external edges belong to the device instance | Accepted | diff --git a/docs/architecture/provisioning/reference.md b/docs/architecture/provisioning/reference.md index 9b2d512..ed4c971 100644 --- a/docs/architecture/provisioning/reference.md +++ b/docs/architecture/provisioning/reference.md @@ -51,8 +51,8 @@ a file path string, a `pathlib.Path` (or another `os.PathLike[str]`), or a | `summary()` | Human-readable, redacted before/after listing, bindings, diagnostics, and unresolved work. | | `phases`, `phase(name)` | `PlannedPhase` records (`planned`, `no_change`, `deferred`, `skipped`) with their `PlannedOperation`s and `FieldChange`s. | | `has_changes` | `True` when any phase is `planned` or `deferred`. | -| `fully_resolved` | `False` when local topology work is deferred, an Inventory update requires replanning, or peer connections remain unresolved. | -| `interface_bindings`, `port_bindings`, `connections` | Legacy IP bindings, generic port bindings, and each connection’s endpoints, edge ids, status and unresolved reason. | +| `fully_resolved` | `False` when local topology work is deferred, an Inventory update requires replanning, or peer edges remain unresolved. | +| `interface_bindings`, `port_bindings`, `edges` | Legacy IP bindings, generic port bindings, and each edge’s endpoints, edge ids, status and unresolved reason. | | `diagnostics`, `skipped_sections` | Warnings and sections the scope skipped. | | `source_key`, `scope`, `variants`, `inventory_id`, `topology_target`, `driver_id`, `driver_schema_version`, `processor_type`, `blueprint_digest` | Identity of what was planned. | @@ -66,10 +66,10 @@ replay elsewhere. |---|---| | `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. `partial` with `replan_required` is returned, not raised. | | `ok` | `True` for `succeeded`, `no_change`, and `planned`. | -| `replan_required` | `True` after an Inventory update requiring rediscovery, or when peer connections remain open. Real runs return `partial`; dry runs return `planned`. Create a new plan when discovery is ready. | +| `replan_required` | `True` after an Inventory update requiring rediscovery, or when peer edges remain open. Real runs return `partial`; dry runs return `planned`. Create a new plan when discovery is ready. | | `inventory_id`, `topology_device_id`, `module_id`, `source_key` | Ids involved. Persist `inventory_id` in your source system. | | `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. `deferred` reports work still waiting for discovery or a new plan. | -| `interface_bindings`, `port_bindings`, `connections`, `diagnostics` | Resolved bindings and connection outcomes after execution. Deferred connections retain the reason and available endpoint information. | +| `interface_bindings`, `port_bindings`, `edges`, `diagnostics` | Resolved bindings and edge outcomes after execution. Deferred edges retain the reason and available endpoint information. | | `materialized` | `True` when deferred topology work was computed during apply. | | `verification`, `verification_detail` | `confirmed`, `unconfirmed`, or `not_applicable`. | | `dry_run` | Whether the execution performed checks without writes. | @@ -89,7 +89,7 @@ your source system knows. | `topology` | `DeviceTarget(device_id)` or `ModuleTarget(device_id, module_id)`, using exact Inspect ids. Leave it out to target the Inventory id, including an id this same plan creates. | | `module_position` | Module context you supply for naming and mappings. It is a string, such as `"0"`, `"A1"`, or `"1/2"`. | | `inventory_overrides` | Per-device Inventory settings that are not secrets (`InventorySettings`): `custom_settings`, `generic_settings`, `snmp`, `metadata`, `active`. Applied after the selected variant. | -| `connections` | Concrete external connections: local port, peer device/module and port, direction and managed edge fields. Defaults to `[]`; does not remove existing edges. | +| `edges` | Concrete external edges: local port, peer device/module and port, direction and managed edge fields. Defaults to `[]`; does not remove existing edges. | | `attributes` | JSON facts you own. Scalar leaves are available to naming as `attributes.`, and processors see them on `context.source`. | `ProvisioningDevice.from_inventory(inventory_device, key=None)` binds an existing @@ -223,7 +223,7 @@ plan = engine.plan( | `vertex_processor` | `processor_type` and `params`. The registered processor validates `params`. See [processors.md](./processors.md). | | `vertices` | Explicit overrides. Point at the vertex with `vertex_id`, or with `factory_label` (may contain `{module.position}`) optionally narrowed by `kind` and `direction` (`In`, `Out`, `Internal`, `Undecided`), and supply `fields`. Those fields use the same contract as processor output. Each override must match exactly one vertex. | | `naming` | `device_label`, `device_description`, `endpoint_label`, `endpoint_description`. | -| `port_mapping` | Generic local connection names mapped to ordered lists of `PortSelector` objects. Supports typed inputs and variant overlays. Keys must differ from `ip_vertex_mapping`. | +| `port_mapping` | Generic local port names mapped to ordered lists of `PortSelector` objects. Supports typed inputs and variant overlays. Keys must differ from `ip_vertex_mapping`. | ### Vertex fields @@ -256,12 +256,17 @@ server. The engine does not store what it applied last time. If a variant should turn a stream off, replace a tag set, or clear an endpoint flag, put that new value in the variant. Background: [ADR-003](./decisions/003-managed-fields.md). -### External connections +### External edges -Declare concrete connections on **`ProvisioningDevice.connections`**. The YAML +Declare concrete edges on **`ProvisioningDevice.edges`**. The YAML blueprint provides reusable local port names; peer identities and edge settings are instance facts supplied by the caller. +One `ProvisioningEdge` declaration can produce one or both directed VideoIPath +edges. Its `EdgeState` is available through `plan.edges` and `result.edges`: +`state.edge` contains the declaration, while `state.edge_ids` lists the directed +server edges. + ```yaml schema_version: 1 defaults: @@ -274,15 +279,15 @@ defaults: ```python from videoipath_automation_tool.provisioning import ( DeviceTarget, EdgePatch, PeerEndpoint, PortSelector, - ProvisioningConnection, ProvisioningDevice, ProvisioningEngine, + ProvisioningEdge, ProvisioningDevice, ProvisioningEngine, ) device = ProvisioningDevice( key="device-a", label="device-a", inventory_id="device1", - connections=[ - ProvisioningConnection( + edges=[ + ProvisioningEdge( local="uplink", peer=PeerEndpoint( target=DeviceTarget(device_id="device2"), @@ -294,11 +299,11 @@ device = ProvisioningDevice( ], ) engine = ProvisioningEngine(app) # an existing VideoIPathApp -plan = engine.plan(device, "external-connections.yml", scope="topology") +plan = engine.plan(device, "external-edges.yml", scope="topology") print(plan.summary()) result = plan.apply() -for connection in result.connections: - print(connection.index, connection.status, connection.edge_ids, connection.reason) +for edge in result.edges: + print(edge.index, edge.status, edge.edge_ids, edge.reason) ``` Use actual VideoIPath ids and factory labels in place of the synthetic values. @@ -308,7 +313,7 @@ use a blueprint with Inventory configuration and the default `scope="all"`. `local` can also be a direct selector such as `PortSelector(port_id="device1.dev.0.port-1")`; no mapping is needed then. The selected configuration still needs a topology section (`topology: {}` is -sufficient). `scope="inventory"` skips all connection work. +sufficient). `scope="inventory"` skips all edge work. `PortSelector` takes exactly one of `port_id` or `factory_label`, with optional `kind` to select vertices of a particular kind. A peer can use `ModuleTarget` @@ -321,8 +326,8 @@ either mapping's keys may be used as `local`. Schema-v2 inputs work at selector value, selector object, candidate list or whole mapping positions. Overlays merge mapping keys and replace candidate lists. Input and literal operators retain their existing atomic replacement -semantics. See [the runnable example](../../examples/07_provisioning/03_external_connections.py) -and [its input-enabled blueprint](../../examples/07_provisioning/external-connections.yml). +semantics. See [the runnable example](../../examples/07_provisioning/03_external_edges.py) +and [its input-enabled blueprint](../../examples/07_provisioning/external-edges.yml). | Direction | Behavior relative to the local device | |---|---| @@ -340,12 +345,12 @@ directions fail; internal device structure remains driver-owned. `conflict_priority`, `include_formats`, `exclude_formats`, `bandwidth_weight_factor`, `weight_per_service`, `active`, and `tags`. Omitted fields remain unmanaged, explicit `null` is rejected, and tags support -replacement lists or `TagDelta`. Use two directed connection declarations when +replacement lists or `TagDelta`. Use two directed edge declarations when opposite directions need different settings. Repeated declarations of the same directed edge combine compatible fields and reject conflicting requirements. Missing edges are created; existing edges are updated only where managed values -differ. Other edges are never removed, including when `connections=[]`. +differ. Other edges are never removed, including when `edges=[]`. Device, vertex and edge changes share the topology transaction. Plans check peer/edge baselines and uncommitted edit overlap before writing, and verify written edge fields afterward. @@ -630,7 +635,7 @@ An accepted topology addition is not resubmitted while waiting for visibility. An accepted synchronization with the same pending changes is polled; newly reported changes can trigger another permitted synchronization. A retry after partial discovery binds the known Inventory ID and can plan against incomplete -local topology without recreating the record. Missing peer connections still +local topology without recreating the record. Missing peer edges still require a new plan when the peers become available. An Inventory update that changes address, alternate addresses, credentials, @@ -789,7 +794,7 @@ def to_provisioning_device(record: dict) -> ProvisioningDevice: You keep the decisions around the engine: which blueprint and variant to use, which address role is allowed, where credentials come from, where returned ids -are stored, which concrete connections to supply, and when the job runs. +are stored, which concrete edges to supply, and when the job runs. ## 11. Limits and verification status @@ -810,7 +815,7 @@ Everything below is importable from `videoipath_automation_tool.provisioning`. | Engine | `ProvisioningEngine`, `ProvisioningPlan`, `ProvisioningApp` | | Document | `Blueprint`, `BlueprintConfiguration`, `InputDefinition`, `InputReference`, `LiteralValue`, `InventorySettings`, `CatalogId`, `published_json_schema` | | Device facts | `ProvisioningDevice`, `Credentials`, `AlternativeAddress`, `DeviceTarget`, `ModuleTarget` | -| Connections | `ProvisioningConnection`, `PeerEndpoint`, `PortSelector`, `ConnectionState`, `PortBinding` | +| Edges | `ProvisioningEdge`, `PeerEndpoint`, `PortSelector`, `EdgeState`, `PortBinding` | | Patches | `DevicePatch`, `ModulePatch`, `VertexPatch`, `EdgePatch`, `Coordinates`, `TagDelta` | | Options and results | `ApplyOptions`, `ApplyResult`, `PhaseResult`, `PlannedPhase`, `PlannedOperation`, `FieldChange`, `InterfaceBinding`, `Diagnostic` | | Naming | `NamingScheme`, `DEFAULT_NAMING`, `Text`, `Field`, `Join`, `NameContext`, `NameRenderer` | @@ -831,7 +836,7 @@ aliases. | `BlueprintPlan` | `ProvisioningPlan` | | `BlueprintApp` | `ProvisioningApp` | | `BlueprintDevice` | `ProvisioningDevice` | -| `BlueprintConnection` | `ProvisioningConnection` | +| `BlueprintConnection` | `ProvisioningEdge` | | `BlueprintError`, `BlueprintValidationError`, `BlueprintTargetError`, `BlueprintCapabilityError`, `BlueprintConflictError`, `BlueprintApplyError` | The same names with the `Provisioning` prefix | | Logger `videoipath_automation_tool_blueprints` | `videoipath_automation_tool_provisioning` | @@ -840,6 +845,24 @@ and `blueprint_digest` plan field keep their names. Documents use `schema_version: 1`. The packaged schema is `videoipath_automation_tool/provisioning/schemas/blueprint-v1.schema.json`. +### Edge names + +The former Connection names are removed without compatibility aliases. Update +constructors, attribute access and serialized field names: + +| Former API | Current API | +|---|---| +| `ProvisioningConnection` | `ProvisioningEdge` | +| `ProvisioningDevice.connections` / `connections=` | `ProvisioningDevice.edges` / `edges=` | +| `ProvisioningPlan.connections` | `ProvisioningPlan.edges` | +| `ApplyResult.connections` | `ApplyResult.edges` | +| `ConnectionState` | `EdgeState` | +| `ConnectionState.connection` | `EdgeState.edge` | + +`connections=` is rejected by `ProvisioningDevice`. Blueprint YAML is unchanged; +concrete edges still belong to the device instance. Direction selection, managed +fields and deferred peers keep the same behavior. `edges=[]` removes nothing. + ### Readiness options `ApplyOptions.discovery_timeout` has been removed without an alias. Use diff --git a/docs/examples/07_provisioning/03_external_connections.py b/docs/examples/07_provisioning/03_external_edges.py similarity index 75% rename from docs/examples/07_provisioning/03_external_connections.py rename to docs/examples/07_provisioning/03_external_edges.py index e4c6be3..699564b 100644 --- a/docs/examples/07_provisioning/03_external_connections.py +++ b/docs/examples/07_provisioning/03_external_edges.py @@ -1,6 +1,6 @@ -"""Preview connections between existing devices using a reusable port mapping. +"""Preview edges between existing devices using a reusable port mapping. -Connection facts and edge settings belong to the device instance. The blueprint +Edge facts and edge settings belong to the device instance. The blueprint maps reusable local port names. Set VideoIPath credentials through the normal VIPAT_* environment variables and replace the synthetic device ids/port labels. Only planning runs by default; uncomment plan.apply() to execute the changes. @@ -16,8 +16,8 @@ EdgePatch, PeerEndpoint, PortSelector, - ProvisioningConnection, ProvisioningDevice, + ProvisioningEdge, ProvisioningEngine, ) @@ -28,8 +28,8 @@ def main() -> None: key="device-a", label="device-a", inventory_id="device1", - connections=[ - ProvisioningConnection( + edges=[ + ProvisioningEdge( local="uplink", peer=PeerEndpoint( target=DeviceTarget(device_id="device2"), @@ -40,13 +40,13 @@ def main() -> None: ), ], ) - blueprint = Path(__file__).with_name("external-connections.yml") + blueprint = Path(__file__).with_name("external-edges.yml") plan = engine.plan(device, blueprint, scope="topology") print(plan.summary()) # result = plan.apply() - # for connection in result.connections: - # print(connection.index, connection.status, connection.edge_ids, connection.reason) + # for edge in result.edges: + # print(edge.index, edge.status, edge.edge_ids, edge.reason) # A missing peer gives status="partial", replan_required=True. Plan again # when it is available. Existing unrelated edges are never removed. diff --git a/docs/examples/07_provisioning/external-connections.yml b/docs/examples/07_provisioning/external-edges.yml similarity index 100% rename from docs/examples/07_provisioning/external-connections.yml rename to docs/examples/07_provisioning/external-edges.yml diff --git a/docs/examples/README.md b/docs/examples/README.md index 7d9b2f0..751875d 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -107,4 +107,4 @@ declarative device configuration through the SDK. - [02_custom_processor_and_naming.py](07_provisioning/02_custom_processor_and_naming.py) — register a custom vertex processor and a custom endpoint naming convention; device or module scope. -- [03_external_connections.py](07_provisioning/03_external_connections.py) — preview external edges supplied on a device, using [external-connections.yml](07_provisioning/external-connections.yml) for generic port mapping. +- [03_external_edges.py](07_provisioning/03_external_edges.py) — preview external edges supplied on a device, using [external-edges.yml](07_provisioning/external-edges.yml) for generic port mapping. diff --git a/docs/getting-started-guide/05_Provisioning.md b/docs/getting-started-guide/05_Provisioning.md index 6fafc06..4d5fd81 100644 --- a/docs/getting-started-guide/05_Provisioning.md +++ b/docs/getting-started-guide/05_Provisioning.md @@ -63,8 +63,8 @@ the apply writes nothing. ### External edges -Supply concrete connections through `ProvisioningDevice.connections`. Each -`ProvisioningConnection` contains a local port (a mapping key or `PortSelector`), +Supply concrete edges through `ProvisioningDevice.edges`. Each +`ProvisioningEdge` contains a local port (a mapping key or `PortSelector`), a `PeerEndpoint` identifying the other device/module and port, and optional `EdgePatch` settings. The blueprint's `defaults.topology.port_mapping` maps reusable local names to discovered ports. `plan(...).apply()` creates or updates @@ -72,11 +72,11 @@ the requested edges along with device and vertex settings. `direction="auto"` derives one or both directions from the actual port vertices. Existing unrelated edges remain unchanged. Missing peers are reported per -connection as `deferred`; the result is `partial` with `replan_required=True`. +edge as `deferred`; the result is `partial` with `replan_required=True`. Build a new plan after the peer becomes available. -See the [complete example](../examples/07_provisioning/03_external_connections.py) -and [connection reference](../architecture/provisioning/reference.md#external-connections). +See the [complete example](../examples/07_provisioning/03_external_edges.py) +and [edge reference](../architecture/provisioning/reference.md#external-edges). ## 3. Writing a blueprint diff --git a/src/videoipath_automation_tool/provisioning/__init__.py b/src/videoipath_automation_tool/provisioning/__init__.py index 0062de0..f60dfca 100644 --- a/src/videoipath_automation_tool/provisioning/__init__.py +++ b/src/videoipath_automation_tool/provisioning/__init__.py @@ -33,13 +33,13 @@ Blueprint, BlueprintConfiguration, CatalogId, - ConnectionState, Coordinates, Credentials, DevicePatch, DeviceTarget, Diagnostic, EdgePatch, + EdgeState, EndpointIdentity, FieldChange, InterfaceBinding, @@ -53,8 +53,8 @@ PortBinding, PortSelector, ProcessorResult, - ProvisioningConnection, ProvisioningDevice, + ProvisioningEdge, TagDelta, VertexEdit, VertexPatch, @@ -90,7 +90,6 @@ "Blueprint", "BlueprintConfiguration", "CatalogId", - "ConnectionState", "Coordinates", "Credentials", "DevicePatch", @@ -99,6 +98,7 @@ "Diagnostic", "DriverContext", "EdgePatch", + "EdgeState", "EndpointIdentity", "Field", "FieldChange", @@ -132,8 +132,8 @@ "ProvisioningApplyError", "ProvisioningCapabilityError", "ProvisioningConflictError", - "ProvisioningConnection", "ProvisioningDevice", + "ProvisioningEdge", "ProvisioningEngine", "ProvisioningError", "ProvisioningPlan", diff --git a/src/videoipath_automation_tool/provisioning/connections.py b/src/videoipath_automation_tool/provisioning/edges.py similarity index 86% rename from src/videoipath_automation_tool/provisioning/connections.py rename to src/videoipath_automation_tool/provisioning/edges.py index ef4494b..e245d3b 100644 --- a/src/videoipath_automation_tool/provisioning/connections.py +++ b/src/videoipath_automation_tool/provisioning/edges.py @@ -14,13 +14,13 @@ TopologyNotReadyError, ) from videoipath_automation_tool.provisioning.models import ( - ConnectionState, EdgePatch, + EdgeState, FieldChange, PlannedOperation, PortBinding, PortSelector, - ProvisioningConnection, + ProvisioningEdge, TagDelta, ) @@ -40,10 +40,10 @@ class EdgeWork(BaseModel): operation: PlannedOperation | None = None -class ConnectionWork(BaseModel): +class EdgeBatchWork(BaseModel): model_config = ConfigDict(frozen=True) - states: list[ConnectionState] = Field(default_factory=list) + states: list[EdgeState] = Field(default_factory=list) edges: dict[str, EdgeWork] = Field(default_factory=dict) @property @@ -116,66 +116,66 @@ def resolve_port(scope: ScopeData, candidates: list[PortSelector], *, key: str) raise ProvisioningTargetError(f"Port '{key}' has no match in '{scope.module_id or scope.device_id}'.") -def resolve_connections( - connections: list[ProvisioningConnection], +def resolve_edges( + edges: list[ProvisioningEdge], scope: ScopeData, peers: dict[int, ScopeData | None], bindings: list[PortBinding], -) -> ConnectionWork: +) -> EdgeBatchWork: """Resolve concrete endpoints; an absent peer stays deferred until a new plan.""" by_key = {binding.key: binding for binding in bindings} - states: list[ConnectionState] = [] - edges: dict[str, EdgeWork] = {} - for index, connection in enumerate(connections): - if connection.peer.target.device_id == scope.device_id: - raise ProvisioningTargetError("Connections must target another device; internal topology is driver-owned.") - if isinstance(connection.local, str): - if connection.local not in by_key: - raise ProvisioningValidationError(f"Unknown local port mapping '{connection.local}'.") - local = by_key[connection.local] + states: list[EdgeState] = [] + directed_edges: dict[str, EdgeWork] = {} + for index, edge in enumerate(edges): + if edge.peer.target.device_id == scope.device_id: + raise ProvisioningTargetError("Edges must target another device; internal topology is driver-owned.") + if isinstance(edge.local, str): + if edge.local not in by_key: + raise ProvisioningValidationError(f"Unknown local port mapping '{edge.local}'.") + local = by_key[edge.local] else: - local = resolve_port(scope, [connection.local], key=f"connections.{index}.local") + local = resolve_port(scope, [edge.local], key=f"edges.{index}.local") peer_scope = peers[index] if peer_scope is None or not peer_scope.ports: states.append( - ConnectionState( + EdgeState( index=index, - connection=connection, + edge=edge, local=local, status="deferred", reason="Peer device, module or ports are not discovered in topology; create a new plan when available.", ) ) continue - peer = resolve_port(peer_scope, [connection.peer.port], key=f"connections.{index}.peer") - pairs = _directed_pairs(local, peer, connection.direction) + peer = resolve_port(peer_scope, [edge.peer.port], key=f"edges.{index}.peer") + pairs = _directed_pairs(local, peer, edge.direction) edge_ids: list[str] = [] for from_vertex, to_vertex in pairs: edge_id = f"{from_vertex}::{to_vertex}" edge_ids.append(edge_id) - fields = connection.fields - if edge_id in edges: - managed = edges[edge_id].fields.managed() + fields = edge.fields + if edge_id in directed_edges: + managed = directed_edges[edge_id].fields.managed() for name, value in fields.managed().items(): if name in managed and managed[name] != value: raise ProvisioningValidationError(f"Conflicting '{name}' requirements for edge '{edge_id}'.") managed[name] = value fields = EdgePatch.model_validate(managed) - edges[edge_id] = EdgeWork(from_vertex=from_vertex, to_vertex=to_vertex, fields=fields) + directed_edges[edge_id] = EdgeWork(from_vertex=from_vertex, to_vertex=to_vertex, fields=fields) states.append( - ConnectionState( + EdgeState( index=index, - connection=connection, + edge=edge, local=local, peer=peer, edge_ids=edge_ids, status="planned", ) ) - return ConnectionWork(states=states, edges=edges) + return EdgeBatchWork(states=states, edges=directed_edges) -def compare_edges(work: ConnectionWork, current: dict[str, InspectApiEdgeForm]) -> ConnectionWork: +def compare_edges(work: EdgeBatchWork, current: dict[str, InspectApiEdgeForm]) -> EdgeBatchWork: """Compare only managed fields while keeping full baselines for conflict checks.""" edges: dict[str, EdgeWork] = {} for edge_id, edge in work.edges.items(): @@ -232,7 +232,7 @@ def compare_edges(work: ConnectionWork, current: dict[str, InspectApiEdgeForm]) ) for state in work.states ] - return ConnectionWork(states=states, edges=edges) + return EdgeBatchWork(states=states, edges=edges) def edge_value(form: InspectApiEdgeForm, path: str) -> Any: diff --git a/src/videoipath_automation_tool/provisioning/engine.py b/src/videoipath_automation_tool/provisioning/engine.py index 51b1501..8723c6f 100644 --- a/src/videoipath_automation_tool/provisioning/engine.py +++ b/src/videoipath_automation_tool/provisioning/engine.py @@ -65,9 +65,9 @@ ApplyOptions, ApplyResult, Blueprint, - ConnectionState, DeviceTarget, Diagnostic, + EdgeState, InterfaceBinding, ModuleTarget, PhaseName, @@ -119,7 +119,7 @@ class ProvisioningPlan(BaseModel): topology membership, and pending synchronization are materialized during ``apply()`` from the captured configuration. A topology-affecting Inventory update is not: ``apply()`` stops after Inventory and the result asks for a new plan. Missing peers stay deferred until a new plan; - the remaining local configuration and resolvable connections can still be applied. + the remaining local configuration and resolvable edges can still be applied. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -136,7 +136,7 @@ class ProvisioningPlan(BaseModel): phases: list[PlannedPhase] interface_bindings: list[InterfaceBinding] port_bindings: list[PortBinding] = Field(default_factory=list) - connections: list[ConnectionState] = Field(default_factory=list) + edges: list[EdgeState] = Field(default_factory=list) diagnostics: list[Diagnostic] skipped_sections: dict[str, str] fully_resolved: bool @@ -184,19 +184,13 @@ def summary(self) -> str: lines.append(f"interface {binding.key} -> port {binding.port_id} ({binding.candidate})") for binding in self.port_bindings: lines.append(f"port {binding.key} -> {binding.port_id} ({binding.candidate})") - for connection in self.connections: - lines.append( - f"connection {connection.index} [{connection.status}]: " - + (connection.reason or ", ".join(connection.edge_ids)) - ) + for edge in self.edges: + lines.append(f"edge {edge.index} [{edge.status}]: " + (edge.reason or ", ".join(edge.edge_ids))) for diagnostic in self.diagnostics: lines.append(f"[{diagnostic.level}] {diagnostic.code}: {diagnostic.message}") if not self.fully_resolved: - if ( - any(connection.status == "deferred" for connection in self.connections) - and not self._captured.topology_deferred - ): - lines.append("Connections remain open: create a new plan when the peer topology is available.") + if any(edge.status == "deferred" for edge in self.edges) and not self._captured.topology_deferred: + lines.append("Edges remain open: create a new plan when the peer topology is available.") elif self._captured.topology_requires_replan: lines.append( "Not fully resolved: apply stops after Inventory; replan once the driver has rediscovered " @@ -348,17 +342,15 @@ def plan( registry=registry, overrides=device.inventory_overrides, ) - if device.connections and scope != "inventory": + if device.edges and scope != "inventory": if resolved.topology is None: - raise ProvisioningValidationError( - "Connections require a topology section in the selected configuration." - ) + raise ProvisioningValidationError("Edges require a topology section in the selected configuration.") keys = set(resolved.topology.config.ip_vertex_mapping or {}) | set( resolved.topology.config.port_mapping or {} ) - for connection in device.connections: - if isinstance(connection.local, str) and connection.local not in keys: - raise ProvisioningValidationError(f"Unknown local port mapping '{connection.local}'.") + for edge in device.edges: + if isinstance(edge.local, str) and edge.local not in keys: + raise ProvisioningValidationError(f"Unknown local port mapping '{edge.local}'.") captured = _Captured( device=device, resolved=resolved, @@ -433,7 +425,7 @@ class _Captured(BaseModel): topology_work: TopologyWork | None = None topology_deferred: bool = False topology_requires_replan: bool = False - connection_peers: dict[int, ScopeData | None] = Field(default_factory=dict) + edge_peers: dict[int, ScopeData | None] = Field(default_factory=dict) @property def source(self) -> SourceFacts: @@ -464,13 +456,9 @@ def build(self) -> ProvisioningPlan: phases: list[PlannedPhase] = [] diagnostics: list[Diagnostic] = [] - if resolved.topology is not None and captured.device.connections: + if resolved.topology is not None and captured.device.edges: captured = captured.model_copy( - update={ - "connection_peers": InspectGateway(self._app.inspect).read_connection_peers( - captured.device.connections - ) - } + update={"edge_peers": InspectGateway(self._app.inspect).read_edge_peers(captured.device.edges)} ) self._captured = captured @@ -501,16 +489,16 @@ def build(self) -> ProvisioningPlan: phases=phases, interface_bindings=list(topology_work.bindings) if topology_work else [], port_bindings=list(topology_work.port_bindings) if topology_work else [], - connections=[state.model_copy(deep=True) for state in topology_work.connections.states] + edges=[state.model_copy(deep=True) for state in topology_work.edge_work.states] if topology_work else [ - ConnectionState(index=index, connection=connection, status="deferred", reason=deferred_reason) - for index, connection in enumerate(captured.device.connections) + EdgeState(index=index, edge=edge, status="deferred", reason=deferred_reason) + for index, edge in enumerate(captured.device.edges) if resolved.topology is not None ], diagnostics=diagnostics + (list(topology_work.diagnostics) if topology_work else []), skipped_sections=dict(resolved.skipped), - fully_resolved=deferred_reason is None and not (topology_work and topology_work.connections.pending), + fully_resolved=deferred_reason is None and not (topology_work and topology_work.edge_work.pending), ) plan._captured = captured plan._executor = self._executor @@ -609,10 +597,8 @@ def _plan_topology( name="topology", status="planned" if work.has_topology_writes - else ("deferred" if work.connections.pending else "no_change"), - reason="Peer connections remain open; create a new plan when available." - if work.connections.pending - else None, + else ("deferred" if work.edge_work.pending else "no_change"), + reason="Peer edges remain open; create a new plan when available." if work.edge_work.pending else None, operations=work.topology_operations, ) ) @@ -699,9 +685,7 @@ def _materialize_topology( inventory=own, allow_label_collisions=captured.options.naming_collisions == "allow", ) - return gateway.plan_connections( - work, scope, captured.device.connections, captured.connection_peers, captured.device.module_position - ) + return gateway.plan_edges(work, scope, captured.device.edges, captured.edge_peers, captured.device.module_position) # --- Internal: execution --- @@ -736,11 +720,11 @@ def __init__(self, executor: _Executor, plan: ProvisioningPlan, *, dry_run: bool self._dry_run = dry_run self._would_write = False self._result = ApplyResult(source_key=plan.source_key, inventory_id=plan.inventory_id, dry_run=dry_run) - self._result.connections = [ + self._result.edges = [ state.model_copy(deep=True, update={"status": "not_run"}) if state.status == "planned" else state.model_copy(deep=True) - for state in plan.connections + for state in plan.edges ] self._phases: dict[str, PhaseResult] = { phase.name: PhaseResult( @@ -857,19 +841,19 @@ def _run_topology(self) -> None: self._enter("topology") self._result.interface_bindings = list(work.bindings) self._result.port_bindings = list(work.port_bindings) - self._result.connections = [ + self._result.edges = [ state.model_copy(deep=True, update={"status": "not_run"}) if state.status == "planned" and not self._dry_run else state.model_copy(deep=True) - for state in work.connections.states + for state in work.edge_work.states ] - self._result.replan_required = work.connections.pending - if work.connections.pending and self._dry_run: + self._result.replan_required = work.edge_work.pending + if work.edge_work.pending and self._dry_run: self._would_write = True self._result.diagnostics.extend(work.diagnostics) - gateway.check_peers(captured.device.connections, captured.connection_peers) - gateway.check_edges(work.connections) + gateway.check_peers(captured.device.edges, captured.edge_peers) + gateway.check_edges(work.edge_work) overlap = gateway.staged_edit_keys() & work.touched_keys() if overlap: @@ -887,20 +871,20 @@ def _run_topology(self) -> None: phase.status = "completed" phase.operations = list(work.topology_operations) self._wrote.append("topology") - self._result.connections = [ + self._result.edges = [ state.model_copy(update={"status": "completed"}) if state.status == "not_run" else state - for state in self._result.connections + for state in self._result.edges ] else: - phase.status = "deferred" if work.connections.pending else "no_change" - if work.connections.pending: - phase.message = "Peer connections remain open; create a new plan when available." + phase.status = "deferred" if work.edge_work.pending else "no_change" + if work.edge_work.pending: + phase.message = "Peer edges remain open; create a new plan when available." self._run_module_tags(gateway, work) if "topology" in self._wrote or "module_tags" in self._wrote: try: self._mismatches.extend(verify_topology(gateway.read_scope(target), work)) - self._mismatches.extend(gateway.verify_edges(work.connections)) + self._mismatches.extend(gateway.verify_edges(work.edge_work)) except Exception as exc: # noqa: BLE001 - verification never turns an applied write into a failure self._verification_errors.append(f"topology read-back failed: {type(exc).__name__}: {exc}") finally: @@ -1088,11 +1072,11 @@ def _fail(self, exc: Exception) -> None: phase.status = "unknown" if self._unknown else "failed" phase.message = f"{type(exc).__name__}: {exc}" if name == "topology": - self._result.connections = [ + self._result.edges = [ state.model_copy(update={"status": "unknown" if self._unknown else "failed", "reason": phase.message}) if state.status == "not_run" else state - for state in self._result.connections + for state in self._result.edges ] if self._unknown: self._result.status = "unknown" diff --git a/src/videoipath_automation_tool/provisioning/inspect.py b/src/videoipath_automation_tool/provisioning/inspect.py index b13a64a..2646257 100644 --- a/src/videoipath_automation_tool/provisioning/inspect.py +++ b/src/videoipath_automation_tool/provisioning/inspect.py @@ -28,11 +28,11 @@ InspectApiSingleVertexInfo, ) from videoipath_automation_tool.apps.inspect.model.tags import module_resource_id -from videoipath_automation_tool.provisioning.connections import ( - ConnectionWork, +from videoipath_automation_tool.provisioning.edges import ( + EdgeBatchWork, compare_edges, edge_value, - resolve_connections, + resolve_edges, resolve_port, same_edge_value, ) @@ -57,7 +57,7 @@ PortBinding, PortSelector, ProcessorResult, - ProvisioningConnection, + ProvisioningEdge, TagDelta, TopologyTarget, ) @@ -130,14 +130,14 @@ class TopologyWork(BaseModel): expected_module_tags: list[str] | None = None bindings: list[InterfaceBinding] = Field(default_factory=list) port_bindings: list[PortBinding] = Field(default_factory=list) - connections: ConnectionWork = Field(default_factory=ConnectionWork) + edge_work: EdgeBatchWork = Field(default_factory=EdgeBatchWork) diagnostics: list[Diagnostic] = Field(default_factory=list) topology_operations: list[PlannedOperation] = Field(default_factory=list) module_operations: list[PlannedOperation] = Field(default_factory=list) @property def has_topology_writes(self) -> bool: - return bool(self.device_intents or self.vertex_intents or self.connections.operations) + return bool(self.device_intents or self.vertex_intents or self.edge_work.operations) @property def has_module_writes(self) -> bool: @@ -149,7 +149,7 @@ def touched_keys(self) -> set[tuple[str, str]]: keys.add(("device", self.device_id)) if self.has_module_writes and self.module_id is not None: keys.add(("module", self.module_id)) - keys.update(self.connections.touched_keys()) + keys.update(self.edge_work.touched_keys()) return keys @@ -300,20 +300,20 @@ def read_peer_scope(self, target: TopologyTarget) -> ScopeData | None: scope = self.read_scope(target) return scope if scope.ports else None - def read_connection_peers(self, connections: list[ProvisioningConnection]) -> dict[int, ScopeData | None]: + def read_edge_peers(self, edges: list[ProvisioningEdge]) -> dict[int, ScopeData | None]: cache: dict[tuple[str, str | None], ScopeData | None] = {} peers: dict[int, ScopeData | None] = {} - for index, connection in enumerate(connections): - target = connection.peer.target + for index, edge in enumerate(edges): + target = edge.peer.target key = (target.device_id, target.module_id if isinstance(target, ModuleTarget) else None) if key not in cache: cache[key] = self.read_peer_scope(target) peers[index] = cache[key] if peers[index] is not None: - resolve_port(peers[index], [connection.peer.port], key=f"connections.{index}.peer") + resolve_port(peers[index], [edge.peer.port], key=f"edges.{index}.peer") return peers - def check_peers(self, connections: list[ProvisioningConnection], peers: dict[int, ScopeData | None]) -> None: + def check_peers(self, edges: list[ProvisioningEdge], peers: dict[int, ScopeData | None]) -> None: checked: set[tuple[str, str | None]] = set() for index, baseline in peers.items(): if baseline is None: @@ -321,7 +321,7 @@ def check_peers(self, connections: list[ProvisioningConnection], peers: dict[int key = (baseline.device_id, baseline.module_id) if key in checked: continue - current = self.read_peer_scope(connections[index].peer.target) + current = self.read_peer_scope(edges[index].peer.target) if current is None or current.fingerprint != baseline.fingerprint: raise ProvisioningConflictError( f"Peer topology '{baseline.module_id or baseline.device_id}' changed; replan." @@ -333,21 +333,19 @@ def read_edges(self, edge_ids: list[str]) -> dict[str, InspectApiEdgeForm]: return {} return {key: item.edge for key, item in self._api.lookup_edges(edge_ids).data.items()} - def plan_connections( + def plan_edges( self, work: TopologyWork, scope: ScopeData, - connections: list[ProvisioningConnection], + edges: list[ProvisioningEdge], peers: dict[int, ScopeData | None], module_position: str | None, ) -> TopologyWork: resolved_inputs = [ - connection.model_copy( - update={"local": _port_selector(connection.local, module_position, f"connections.{index}.local")} - ) - if isinstance(connection.local, PortSelector) - else connection - for index, connection in enumerate(connections) + edge.model_copy(update={"local": _port_selector(edge.local, module_position, f"edges.{index}.local")}) + if isinstance(edge.local, PortSelector) + else edge + for index, edge in enumerate(edges) ] legacy = [ PortBinding( @@ -357,22 +355,22 @@ def plan_connections( ) for binding in work.bindings ] - pending = resolve_connections(resolved_inputs, scope, peers, [*legacy, *work.port_bindings]) + pending = resolve_edges(resolved_inputs, scope, peers, [*legacy, *work.port_bindings]) compared = compare_edges(pending, self.read_edges(list(pending.edges))) return work.model_copy( update={ - "connections": compared, + "edge_work": compared, "topology_operations": [*work.topology_operations, *compared.operations], } ) - def check_edges(self, work: ConnectionWork) -> None: + def check_edges(self, work: EdgeBatchWork) -> None: current = self.read_edges(list(work.edges)) for edge_id, edge in work.edges.items(): if current.get(edge_id) != edge.baseline: raise ProvisioningConflictError(f"Edge '{edge_id}' changed since planning; create a new plan.") - def verify_edges(self, work: ConnectionWork) -> list[str]: + def verify_edges(self, work: EdgeBatchWork) -> list[str]: written = {key: edge for key, edge in work.edges.items() if edge.operation is not None} current = self.read_edges(list(written)) return [ @@ -400,14 +398,14 @@ def commit(self, work: TopologyWork) -> Any: tx.update_device(work.device_id, intents=dict(work.device_intents)) for vertex_id, intents in sorted(work.vertex_intents.items()): tx.update_vertex(vertex_id, intents=dict(intents)) - for edge_id, edge in work.connections.edges.items(): + for edge_id, edge in work.edge_work.edges.items(): if edge.operation is None: continue if edge.baseline is None: tx.connect(edge.from_vertex, edge.to_vertex, bidirectional=False, **edge.intents) else: tx.update_edge(edge_id, intents=dict(edge.intents)) - self.check_edges(work.connections) + self.check_edges(work.edge_work) return tx.commit() def assign_tag(self, tag_id: str, module_id: str) -> None: diff --git a/src/videoipath_automation_tool/provisioning/models.py b/src/videoipath_automation_tool/provisioning/models.py index 94098d9..e1ac0d2 100644 --- a/src/videoipath_automation_tool/provisioning/models.py +++ b/src/videoipath_automation_tool/provisioning/models.py @@ -183,7 +183,7 @@ def _one_selector(self) -> PortSelector: class EdgePatch(_PatchModel): - """Managed fields of a concrete connection, applied to each requested direction.""" + """Managed fields of a concrete edge, applied to each requested direction.""" label: str | None = None description: str | None = None @@ -457,8 +457,11 @@ class PeerEndpoint(_StrictModel): port: PortSelector -class ProvisioningConnection(_StrictModel): - """Concrete external connection; directions are relative to the local device.""" +class ProvisioningEdge(_StrictModel): + """Concrete external edge declaration; directions are relative to the local device. + + One declaration can resolve to one or both directed VideoIPath edges. + """ local: NonEmptyStr | PortSelector peer: PeerEndpoint @@ -476,7 +479,7 @@ class ProvisioningDevice(_StrictModel): set and also replaces per-address credentials: a plain string carries none, so an existing credential for that address is cleared unless it is repeated as an ``AlternativeAddress``. ``attributes`` are caller-owned facts available to naming (scalar leaves) and processors. - ``connections`` declares concrete external connections and their managed edge fields. + ``edges`` declares concrete external edges and their managed edge fields. Missing peer discovery is reported as deferred; it never creates the peer device. """ @@ -491,7 +494,7 @@ class ProvisioningDevice(_StrictModel): module_position: NonEmptyStr | None = None inventory_overrides: InventorySettings | None = None attributes: dict[str, JsonValue] = Field(default_factory=dict) - connections: list[ProvisioningConnection] = Field(default_factory=list) + edges: list[ProvisioningEdge] = Field(default_factory=list) @model_validator(mode="after") def _unique_addresses(self) -> ProvisioningDevice: @@ -636,11 +639,11 @@ class PortBinding(InterfaceBinding): module_id: str | None = None -class ConnectionState(_StrictModel): - """One input connection's resolution and execution state, including unresolved work.""" +class EdgeState(_StrictModel): + """One input edge's resolution and execution state, including unresolved work.""" index: int - connection: ProvisioningConnection + edge: ProvisioningEdge local: PortBinding | None = None peer: PortBinding | None = None edge_ids: list[str] = Field(default_factory=list) @@ -663,7 +666,7 @@ class ApplyResult(BaseModel): run and ``no_change`` otherwise, and write phases report ``planned`` / ``deferred``. ``replan_required`` means a topology-affecting Inventory update finished and apply stopped, - or some connections still have undiscovered peers. + or some edges still have undiscovered peers. ``status`` is ``partial`` (``planned`` on a dry run) and the call does not raise. Plan again after the driver has rediscovered the device or the peer topology is available. """ @@ -671,7 +674,7 @@ class ApplyResult(BaseModel): status: Literal["succeeded", "no_change", "failed", "partial", "unknown", "planned"] = "no_change" dry_run: bool = False replan_required: bool = False - """A new plan is needed after Inventory rediscovery or to resolve open peer connections.""" + """A new plan is needed after Inventory rediscovery or to resolve open peer edges.""" source_key: str inventory_id: str | None = None topology_device_id: str | None = None @@ -679,7 +682,7 @@ class ApplyResult(BaseModel): phases: list[PhaseResult] = Field(default_factory=list) interface_bindings: list[InterfaceBinding] = Field(default_factory=list) port_bindings: list[PortBinding] = Field(default_factory=list) - connections: list[ConnectionState] = Field(default_factory=list) + edges: list[EdgeState] = Field(default_factory=list) materialized: bool = False diagnostics: list[Diagnostic] = Field(default_factory=list) verification: Literal["confirmed", "unconfirmed", "not_applicable"] = "not_applicable" @@ -791,13 +794,13 @@ def _friendly_message(error: Any) -> str: "Blueprint", "BlueprintConfiguration", "CatalogId", - "ConnectionState", "Coordinates", "Credentials", "DevicePatch", "DeviceTarget", "Diagnostic", "EdgePatch", + "EdgeState", "EndpointIdentity", "FieldChange", "GenericSettings", @@ -813,8 +816,8 @@ def _friendly_message(error: Any) -> str: "PortBinding", "PortSelector", "ProcessorResult", - "ProvisioningConnection", "ProvisioningDevice", + "ProvisioningEdge", "Scope", "SnmpSelection", "TagDelta", diff --git a/tests/e2e/provisioning/test_connections.py b/tests/e2e/provisioning/test_edges.py similarity index 84% rename from tests/e2e/provisioning/test_connections.py rename to tests/e2e/provisioning/test_edges.py index 6771d20..ed0b44f 100644 --- a/tests/e2e/provisioning/test_connections.py +++ b/tests/e2e/provisioning/test_edges.py @@ -12,7 +12,7 @@ EdgePatch, PeerEndpoint, PortSelector, - ProvisioningConnection, + ProvisioningEdge, ) from ..helpers import edges_between @@ -25,7 +25,7 @@ pytestmark = pytest.mark.e2e -def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> None: +def test_directed_edges_and_managed_updates(live: LiveProvisioning) -> None: local = live.onboard("LINK-A") peer = live.onboard("LINK-B") output, input_port = _ports(live, local) @@ -35,8 +35,8 @@ def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> Non blueprint = Blueprint.from_dict(document) local = local.model_copy( update={ - "connections": [ - ProvisioningConnection( + "edges": [ + ProvisioningEdge( local="send", peer=PeerEndpoint( target=DeviceTarget(device_id=peer.inventory_id), port=PortSelector(port_id=peer_input.id) @@ -44,7 +44,7 @@ def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> Non direction="outgoing", fields=EdgePatch(label=f"{local.label}-out", weight=7, capacity=1000, tags=[live.tag]), ), - ProvisioningConnection( + ProvisioningEdge( local=PortSelector(port_id=input_port.id), peer=PeerEndpoint( target=DeviceTarget(device_id=peer.inventory_id), port=PortSelector(port_id=peer_output.id) @@ -59,15 +59,15 @@ def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> Non plan = live.plan(local, blueprint=blueprint, scope="topology") assert plan.fully_resolved assert plan.port_bindings[0].port_id == output.id - assert [c.status for c in plan.connections] == ["planned", "planned"] + assert [c.status for c in plan.edges] == ["planned", "planned"] assert plan.apply(dry_run=True).status == "planned" assert_no_writes(writes) live.app.inspect.refresh() assert edges_between(live.app, local.inventory_id, peer.inventory_id) == [] result = plan.apply() assert_confirmed(result) - assert [c.status for c in result.connections] == ["completed", "completed"] - edge_ids = {edge_id for c in result.connections for edge_id in c.edge_ids} + assert [c.status for c in result.edges] == ["completed", "completed"] + edge_ids = {edge_id for c in result.edges for edge_id in c.edge_ids} live.app.inspect.refresh() edges = edges_between(live.app, local.inventory_id, peer.inventory_id) assert {e.id for e in edges} == edge_ids @@ -75,7 +75,7 @@ def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> Non (output.id, peer_input.id), (peer_output.id, input_port.id), } - outbound_id = result.connections[0].edge_ids[0] + outbound_id = result.edges[0].edge_ids[0] outbound = next(e for e in edges if e.id == outbound_id) assert (outbound.label, outbound.weight, outbound.capacity) == (f"{local.label}-out", 7, 1000) assert live.tag in outbound.tags @@ -88,12 +88,12 @@ def test_directed_connections_and_managed_updates(live: LiveProvisioning) -> Non with live.app.inspect.transaction() as tx: tx.update_edge(outbound_id, description="E2E preserve-edge-description") tx.commit() - unchanged_id = result.connections[1].edge_ids[0] + unchanged_id = result.edges[1].edge_ids[0] live.app.inspect.refresh() unchanged = next(e for e in live.app.inspect.edges if e.id == unchanged_id) unchanged_before = (unchanged.label, unchanged.weight, unchanged.tags) - connection = local.connections[0].model_copy(update={"fields": EdgePatch(weight=0, active=False)}) - updated = local.model_copy(update={"connections": [connection]}) + edge = local.edges[0].model_copy(update={"fields": EdgePatch(weight=0, active=False)}) + updated = local.model_copy(update={"edges": [edge]}) assert_confirmed(live.plan(updated, blueprint=blueprint, scope="topology").apply()) live.app.inspect.refresh() edges = {e.id: e for e in edges_between(live.app, local.inventory_id, peer.inventory_id)} @@ -110,8 +110,8 @@ def test_missing_peer_requires_a_new_plan(live: LiveProvisioning) -> None: output, _ = _ports(live, local) local = local.model_copy( update={ - "connections": [ - ProvisioningConnection( + "edges": [ + ProvisioningEdge( local=PortSelector(port_id=output.id), peer=PeerEndpoint( target=DeviceTarget(device_id=peer.inventory_id), @@ -124,18 +124,18 @@ def test_missing_peer_requires_a_new_plan(live: LiveProvisioning) -> None: } ) plan = live.plan(local, scope="topology") - assert not plan.fully_resolved and plan.connections[0].status == "deferred" + assert not plan.fully_resolved and plan.edges[0].status == "deferred" with observe_writes(live.app) as writes: preview = plan.apply(dry_run=True) assert preview.status == "planned" and preview.replan_required assert_no_writes(writes) - # Availability after planning must not silently change the reviewed connection plan. + # Availability after planning must not silently change the reviewed edge plan. assert_confirmed(live.plan(peer, scope="topology").apply()) with observe_writes(live.app) as writes: pending = plan.apply() assert pending.status == "partial" and pending.replan_required - assert pending.connections[0].status == "deferred" + assert pending.edges[0].status == "deferred" assert_no_writes(writes) live.app.inspect.refresh() assert edges_between(live.app, local.inventory_id, peer.inventory_id) == [] @@ -144,9 +144,9 @@ def test_missing_peer_requires_a_new_plan(live: LiveProvisioning) -> None: assert not resolved.replan_required live.app.inspect.refresh() assert {e.id for e in edges_between(live.app, local.inventory_id, peer.inventory_id)} == set( - resolved.connections[0].edge_ids + resolved.edges[0].edge_ids ) - assert len(resolved.connections[0].edge_ids) == 1 + assert len(resolved.edges[0].edge_ids) == 1 def _ports(live: LiveProvisioning, device: ProvisioningDevice) -> tuple[InspectPort, InspectPort]: diff --git a/tests/e2e/provisioning/test_recovery.py b/tests/e2e/provisioning/test_recovery.py index 0a01cb6..2f73cfc 100644 --- a/tests/e2e/provisioning/test_recovery.py +++ b/tests/e2e/provisioning/test_recovery.py @@ -18,7 +18,7 @@ ProvisioningApplyError, ProvisioningCapabilityError, ProvisioningConflictError, - ProvisioningConnection, + ProvisioningEdge, ) from .helpers import ( @@ -122,12 +122,12 @@ def test_stale_device_plan_preserves_external_change(live: LiveProvisioning, sco assert description == "E2E external-description" -def test_stale_connection_plan_preserves_external_change(live: LiveProvisioning) -> None: +def test_stale_edge_plan_preserves_external_change(live: LiveProvisioning) -> None: local, peer = live.onboard("STALE-LINK-A"), live.onboard("STALE-LINK-B") local = local.model_copy( update={ - "connections": [ - ProvisioningConnection( + "edges": [ + ProvisioningEdge( local=PortSelector(factory_label="Router Out 11.1"), peer=PeerEndpoint( target=DeviceTarget(device_id=peer.inventory_id), @@ -141,10 +141,8 @@ def test_stale_connection_plan_preserves_external_change(live: LiveProvisioning) ) result = live.plan(local, scope="topology").apply() assert_confirmed(result) - edge_id = result.connections[0].edge_ids[0] - desired = local.model_copy( - update={"connections": [local.connections[0].model_copy(update={"fields": EdgePatch(weight=5)})]} - ) + edge_id = result.edges[0].edge_ids[0] + desired = local.model_copy(update={"edges": [local.edges[0].model_copy(update={"fields": EdgePatch(weight=5)})]}) plan = live.plan(desired, scope="topology") with live.app.inspect.transaction() as tx: tx.update_edge(edge_id, weight=11) diff --git a/tests/provisioning/test_connection_inputs.py b/tests/provisioning/test_edge_inputs.py similarity index 94% rename from tests/provisioning/test_connection_inputs.py rename to tests/provisioning/test_edge_inputs.py index d5c837d..8e24aa8 100644 --- a/tests/provisioning/test_connection_inputs.py +++ b/tests/provisioning/test_edge_inputs.py @@ -7,7 +7,7 @@ import pytest from tests.provisioning.conftest import FakeInspectServer, FakeInventory -from tests.provisioning.test_connections import _connection, _device, _seed +from tests.provisioning.test_edges import _device, _edge, _seed from videoipath_automation_tool.provisioning import Blueprint, ProvisioningEngine, ProvisioningValidationError from videoipath_automation_tool.provisioning.resolution import resolve_blueprint @@ -49,10 +49,10 @@ def test_typed_mapping_input_positions( inputs = {"ports": values[position]} resolved = resolve_blueprint(blueprint, inputs=inputs) assert any("input:ports" in source for source in resolved.topology.provenance.values()) - plan = engine.plan(_device(_connection()), blueprint, inputs=inputs) + plan = engine.plan(_device(_edge()), blueprint, inputs=inputs) inputs["ports"] = "changed" result = plan.apply() - assert result.connections[0].local.port_id == "device1.dev.0.P1" + assert result.edges[0].local.port_id == "device1.dev.0.P1" assert result.ok and len(server.edge_forms) == 2 @@ -132,7 +132,7 @@ def __getattr__(self, key: str) -> Any: with pytest.raises(ProvisioningValidationError, match="exactly one") as error: ProvisioningEngine(NoIO()).plan( - _device(_connection()), blueprint, inputs={"selector": {"port_id": "port-a", "factory_label": "P1"}} + _device(_edge()), blueprint, inputs={"selector": {"port_id": "port-a", "factory_label": "P1"}} ) issue = error.value.issues[0] assert issue.source == "device.yml" and issue.line == 10 diff --git a/tests/provisioning/test_connections.py b/tests/provisioning/test_edges.py similarity index 68% rename from tests/provisioning/test_connections.py rename to tests/provisioning/test_edges.py index 0c10406..2df7e0f 100644 --- a/tests/provisioning/test_connections.py +++ b/tests/provisioning/test_edges.py @@ -1,7 +1,9 @@ -"""Generic connections through the real provisioning engine and Inspect transaction, offline.""" +"""Generic edges through the real provisioning engine and Inspect transaction, offline.""" from __future__ import annotations +import json +from importlib.util import find_spec from types import SimpleNamespace from typing import Any @@ -9,6 +11,7 @@ from pydantic import ValidationError from tests.provisioning.conftest import HEADER, FakeInspectServer, FakeInventory, build_layout, matrox_layout +from videoipath_automation_tool import provisioning from videoipath_automation_tool.apps.inspect.errors import InspectCommitError from videoipath_automation_tool.apps.inspect.model.actions import InspectApiEdgeForm from videoipath_automation_tool.apps.inspect.model.update_topology import InspectApiUpdateTopologyResponse @@ -16,16 +19,18 @@ Blueprint, DeviceTarget, EdgePatch, + EdgeState, ModuleTarget, PeerEndpoint, PortSelector, ProvisioningApplyError, - ProvisioningConnection, ProvisioningDevice, + ProvisioningEdge, ProvisioningEngine, ProvisioningTargetError, ProvisioningValidationError, TagDelta, + models, ) @@ -34,19 +39,17 @@ def _blueprint(**topology: Any) -> Blueprint: return Blueprint.from_dict({"schema_version": 1, "defaults": {"topology": config}}) -def _connection(**kwargs: Any) -> ProvisioningConnection: +def _edge(**kwargs: Any) -> ProvisioningEdge: values = { "local": "uplink", "peer": PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="P1")), **kwargs, } - return ProvisioningConnection(**values) + return ProvisioningEdge(**values) -def _device(*connections: ProvisioningConnection, **kwargs: Any) -> ProvisioningDevice: - return ProvisioningDevice( - key="device-a", label="device-a", inventory_id="device1", connections=list(connections), **kwargs - ) +def _device(*edges: ProvisioningEdge, **kwargs: Any) -> ProvisioningDevice: + return ProvisioningDevice(key="device-a", label="device-a", inventory_id="device1", edges=list(edges), **kwargs) def _seed(inventory: FakeInventory, server: FakeInspectServer, *, peer: bool = True) -> None: @@ -56,24 +59,76 @@ def _seed(inventory: FakeInventory, server: FakeInspectServer, *, peer: bool = T server.install(matrox_layout("device2")) +def test_public_edge_api_and_serialization( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, +) -> None: + _seed(inventory, server) + edge = _edge(fields=EdgePatch(label="edge-a", weight=3)) + device = _device(edge) + assert ProvisioningDevice(key="device-b", label="device-b").edges == [] + assert ProvisioningDevice.model_validate_json(device.model_dump_json(exclude_unset=True)) == device + assert {"ProvisioningEdge", "EdgeState"} <= set(provisioning.__all__) + + plan = engine.plan(device, _blueprint()) + result = plan.apply() + for value in (device, plan, result): + payload = value.model_dump(mode="json") + assert json.loads(value.model_dump_json()) == payload + assert "connections" not in payload and len(payload["edges"]) == 1 + assert not hasattr(value, "connections") + for value in (plan, result): + state = value.edges[0] + assert isinstance(state, EdgeState) and isinstance(state.edge, ProvisioningEdge) + assert state.edge == edge and len(state.edge_ids) == 2 + payload = json.loads(value.model_dump_json())["edges"][0] + assert "connection" not in payload and payload["edge"] == edge.model_dump(mode="json") + assert not hasattr(state, "connection") + assert "edge 0 [planned]" in plan.summary() + assert "connection" not in plan.summary().lower() + + +def test_removed_edge_api_names_have_no_aliases() -> None: + for old_name in ("ProvisioningConnection", "ConnectionState"): + assert not hasattr(provisioning, old_name) + assert not hasattr(models, old_name) + assert find_spec("videoipath_automation_tool.provisioning.connections") is None + + +@pytest.mark.parametrize("value", [[], [_edge().model_dump()]]) +def test_device_rejects_removed_connections_field(value: list[dict[str, Any]]) -> None: + with pytest.raises(ValidationError) as error: + ProvisioningDevice(key="device-a", label="device-a", connections=value) + assert [(issue["loc"], issue["type"]) for issue in error.value.errors()] == [(("connections",), "extra_forbidden")] + + +def test_edge_validation_paths_use_new_field_name() -> None: + fields = _edge().model_dump() + fields["direction"] = "invalid" + with pytest.raises(ValidationError) as error: + ProvisioningDevice(key="device-a", label="device-a", edges=[fields]) + assert error.value.errors()[0]["loc"] == ("edges", 0, "direction") + + def test_bidirectional_create_shared_commit_and_idempotence( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, ) -> None: _seed(inventory, server) - device = _device(_connection(fields=EdgePatch(weight=0, capacity=0, bandwidth=0.0, active=False))) + device = _device(_edge(fields=EdgePatch(weight=0, capacity=0, bandwidth=0.0, active=False))) blueprint = _blueprint(vertices=[{"factory_label": "Video Sender", "fields": {"active": False}}]) plan = engine.plan(device, blueprint) assert not server.writes - assert plan.fully_resolved and len(plan.connections[0].edge_ids) == 2 + assert plan.fully_resolved and len(plan.edges[0].edge_ids) == 2 assert plan.interface_bindings == [] and plan.port_bindings[0].key == "uplink" - assert "connection 0 [planned]" in plan.summary() - assert plan.apply(dry_run=True).connections[0].status == "planned" + assert "edge 0 [planned]" in plan.summary() + assert plan.apply(dry_run=True).edges[0].status == "planned" assert not server.writes result = plan.apply() assert result.status == "succeeded" and result.verification == "confirmed" - assert result.connections[0].status == "completed" + assert result.edges[0].status == "completed" assert len(server.writes) == 1 delta = server.writes[0][1] assert delta["replaceDevices"] and delta["replaceVertices"] and len(delta["replaceEdges"]) == 2 @@ -100,10 +155,10 @@ def test_directed_media_without_direction_suffix( ) peer = PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="media-1")) result = engine.apply( - _device(_connection(local=PortSelector(factory_label="media-1"), peer=peer, direction=direction)), + _device(_edge(local=PortSelector(factory_label="media-1"), peer=peer, direction=direction)), _blueprint(port_mapping={}), ) - assert result.connections[0].edge_ids == ["device1.0.port-1.vertex::device2.0.port-1.vertex"] + assert result.edges[0].edge_ids == ["device1.0.port-1.vertex::device2.0.port-1.vertex"] assert len(server.edge_forms) == 1 @@ -112,10 +167,10 @@ def test_explicit_directions( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, direction: str, count: int ) -> None: _seed(inventory, server) - result = engine.apply(_device(_connection(direction=direction)), _blueprint()) - assert len(result.connections[0].edge_ids) == count + result = engine.apply(_device(_edge(direction=direction)), _blueprint()) + assert len(result.edges[0].edge_ids) == count if direction == "incoming": - assert result.connections[0].edge_ids == ["device2.0.P1.out::device1.0.P1.in"] + assert result.edges[0].edge_ids == ["device2.0.P1.out::device1.0.P1.in"] def test_managed_updates_preserve_other_edges_and_nested_fields( @@ -124,7 +179,7 @@ def test_managed_updates_preserve_other_edges_and_nested_fields( server: FakeInspectServer, ) -> None: _seed(inventory, server) - engine.apply(_device(_connection()), _blueprint()) + engine.apply(_device(_edge()), _blueprint()) for form in server.edge_forms.values(): form["descriptor"]["desc"] = "keep-description" form["fDescriptor"]["label"] = "factory-edge" @@ -142,8 +197,8 @@ def test_managed_updates_preserve_other_edges_and_nested_fields( redundancy_mode="Any", bandwidth_weight_factor=0, ) - result = engine.apply(_device(_connection(fields=fields)), _blueprint()) - for edge_id in result.connections[0].edge_ids: + result = engine.apply(_device(_edge(fields=fields)), _blueprint()) + for edge_id in result.edges[0].edge_ids: form = server.edge_forms[edge_id] assert form["descriptor"] == {"label": "edge-a", "desc": "keep-description"} assert form["fDescriptor"]["label"] == "factory-edge" @@ -151,22 +206,22 @@ def test_managed_updates_preserve_other_edges_and_nested_fields( assert form["weightFactors"] == {"bandwidth": {"weight": 0}, "service": {"max": 43, "weight": 0}} assert form["conflictPri"] == 1 and form["excludeFormats"] == ["format-a"] assert foreign in server.edge_forms - assert engine.apply(_device(_connection(fields=fields)), _blueprint()).status == "no_change" + assert engine.apply(_device(_edge(fields=fields)), _blueprint()).status == "no_change" assert engine.apply(_device(), _blueprint()).status == "no_change" assert len(server.edge_forms) == 3 -def test_duplicate_connections_coalesce_or_reject_conflicts( +def test_duplicate_edges_coalesce_or_reject_conflicts( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, ) -> None: _seed(inventory, server) - connection = _connection(fields=EdgePatch(weight=3)) - plan = engine.plan(_device(connection, connection, _connection(fields=EdgePatch(label="edge-a"))), _blueprint()) + edge = _edge(fields=EdgePatch(weight=3)) + plan = engine.plan(_device(edge, edge, _edge(fields=EdgePatch(label="edge-a"))), _blueprint()) assert len([op for op in plan.phase("topology").operations if op.entity_kind == "edge"]) == 2 with pytest.raises(ProvisioningValidationError, match="Conflicting 'weight'"): - engine.plan(_device(connection, _connection(fields=EdgePatch(weight=4))), _blueprint()) + engine.plan(_device(edge, _edge(fields=EdgePatch(weight=4))), _blueprint()) assert not server.writes @@ -176,16 +231,16 @@ def test_missing_peer_is_partial_and_frozen_until_replan( server: FakeInspectServer, ) -> None: _seed(inventory, server, peer=False) - device, blueprint = _device(_connection()), _blueprint() + device, blueprint = _device(_edge()), _blueprint() plan = engine.plan(device, blueprint) - assert not plan.fully_resolved and plan.connections[0].status == "deferred" + assert not plan.fully_resolved and plan.edges[0].status == "deferred" preview = plan.apply(dry_run=True) assert preview.status == "planned" and preview.replan_required and not server.writes server.install(matrox_layout("device2")) result = plan.apply() assert result.status == "partial" and result.replan_required and not result.ok assert not server.edge_forms - assert result.connections[0].local and result.connections[0].peer is None + assert result.edges[0].local and result.edges[0].peer is None assert engine.apply(device, blueprint).status == "succeeded" @@ -197,21 +252,21 @@ def test_pending_without_any_write_is_still_partial( _seed(inventory, server, peer=False) engine.apply(_device(), _blueprint()) server.writes.clear() - result = engine.apply(_device(_connection()), _blueprint()) + result = engine.apply(_device(_edge()), _blueprint()) assert result.status == "partial" and result.phase("topology").status == "deferred" assert result.verification == "not_applicable" and not server.writes -def test_mixed_ready_and_pending_connections( +def test_mixed_ready_and_pending_edges( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, ) -> None: _seed(inventory, server) missing = PeerEndpoint(target=DeviceTarget(device_id="device3"), port=PortSelector(factory_label="P1")) - result = engine.apply(_device(_connection(), _connection(peer=missing)), _blueprint()) + result = engine.apply(_device(_edge(), _edge(peer=missing)), _blueprint()) assert result.status == "partial" and result.verification == "confirmed" - assert [state.status for state in result.connections] == ["completed", "deferred"] + assert [state.status for state in result.edges] == ["completed", "deferred"] assert len(server.edge_forms) == 2 @@ -227,7 +282,7 @@ def test_undiscovered_peer_module_or_ports_is_pending( server.nodes["device2"]["modules"]["device2.dev.0"]["ports"] = {} target = ModuleTarget(device_id="device2", module_id="device2.dev.1" if missing == "module" else "device2.dev.0") peer = PeerEndpoint(target=target, port=PortSelector(factory_label="P1")) - assert engine.apply(_device(_connection(peer=peer)), _blueprint()).connections[0].status == "deferred" + assert engine.apply(_device(_edge(peer=peer)), _blueprint()).edges[0].status == "deferred" def test_peer_selector_miss_and_transport_error_are_not_pending( @@ -238,15 +293,15 @@ def test_peer_selector_miss_and_transport_error_are_not_pending( ) -> None: _seed(inventory, server) peer = PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="absent")) - with pytest.raises(ProvisioningTargetError, match="no match"): - engine.plan(_device(_connection(peer=peer)), _blueprint()) + with pytest.raises(ProvisioningTargetError, match=r"Port 'edges\.0\.peer' has no match"): + engine.plan(_device(_edge(peer=peer)), _blueprint()) def fail(device_id: str) -> None: raise OSError("read failed") monkeypatch.setattr(server, "get_device_detail", fail) with pytest.raises(OSError, match="read failed"): - engine.plan(_device(_connection()), _blueprint()) + engine.plan(_device(_edge()), _blueprint()) assert not server.writes @@ -260,7 +315,7 @@ def test_stale_peer_and_edge_baselines( dry_run: bool, ) -> None: _seed(inventory, server) - device, blueprint = _device(_connection()), _blueprint() + device, blueprint = _device(_edge()), _blueprint() if change in ("edge_updated", "edge_removed"): engine.apply(device, blueprint) plan = engine.plan(device, blueprint) @@ -281,7 +336,7 @@ def test_stale_peer_and_edge_baselines( @pytest.mark.parametrize("kind", ["edge", "vertex", "module", "device"]) -def test_pending_snapshot_edits_on_connection_entities( +def test_pending_snapshot_edits_on_edge_entities( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, @@ -289,9 +344,9 @@ def test_pending_snapshot_edits_on_connection_entities( kind: str, ) -> None: _seed(inventory, server) - plan = engine.plan(_device(_connection()), _blueprint()) + plan = engine.plan(_device(_edge()), _blueprint()) entity_id = { - "edge": plan.connections[0].edge_ids[0], + "edge": plan.edges[0].edge_ids[0], "vertex": "device2.0.P1.in", "module": "device2.dev.0", "device": "device2", @@ -302,21 +357,21 @@ def test_pending_snapshot_edits_on_connection_entities( assert not server.writes -def test_plan_copies_connections_and_edge_fields( +def test_plan_copies_edges_and_edge_fields( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, ) -> None: _seed(inventory, server) fields = EdgePatch(tags=["tag-a"]) - device = _device(_connection(fields=fields)) + device = _device(_edge(fields=fields)) plan = engine.plan(device, _blueprint()) fields.tags.append("tag-b") - device.connections.clear() - plan.connections[0].connection.fields.tags.append("tag-c") + device.edges.clear() + plan.edges[0].edge.fields.tags.append("tag-c") result = plan.apply() assert all(form["tags"] == ["tag-a"] for form in server.edge_forms.values()) - assert result.connections[0].status == "completed" + assert result.edges[0].status == "completed" def test_local_discovery_materializes_edges( @@ -327,9 +382,9 @@ def test_local_discovery_materializes_edges( inventory.seed("device1") server.discoverable["device1"] = matrox_layout("device1") server.install(matrox_layout("device2")) - plan = engine.plan(_device(_connection()), _blueprint()) + plan = engine.plan(_device(_edge()), _blueprint()) assert not plan.fully_resolved - assert plan.apply(dry_run=True).connections[0].status == "deferred" + assert plan.apply(dry_run=True).edges[0].status == "deferred" assert not server.writes result = plan.apply() assert result.materialized and result.status == "succeeded" and len(server.edge_forms) == 2 @@ -354,8 +409,8 @@ def test_failure_and_verification_status( ) server.fail_commit = InspectCommitError(rejection) with pytest.raises(ProvisioningApplyError) as error: - engine.apply(_device(_connection()), _blueprint()) - assert error.value.result.connections[0].status == "failed" + engine.apply(_device(_edge()), _blueprint()) + assert error.value.result.edges[0].status == "failed" assert not server.edge_forms server.fail_commit = None original = server.update_topology @@ -366,12 +421,12 @@ def discard_edges(delta: Any) -> Any: return result monkeypatch.setattr(server, "update_topology", discard_edges) - result = engine.apply(_device(_connection()), _blueprint()) + result = engine.apply(_device(_edge()), _blueprint()) assert result.status == "succeeded" and result.verification == "unconfirmed" assert "fromId" in result.verification_detail -def test_legacy_ip_mapping_can_supply_connection_without_changing_processor_inputs( +def test_legacy_ip_mapping_can_supply_edge_without_changing_processor_inputs( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, @@ -382,7 +437,7 @@ def test_legacy_ip_mapping_can_supply_connection_without_changing_processor_inpu ip_vertex_mapping={"uplink": ["P1"], "backup": ["P2"]}, vertex_processor={"processor_type": "matrox.convertip.default", "params": {"redundant_streams": True}}, ) - plan = engine.plan(_device(_connection()), blueprint) + plan = engine.plan(_device(_edge()), blueprint) assert [binding.key for binding in plan.interface_bindings] == ["uplink", "backup"] assert [binding.key for binding in plan.port_bindings] == ["media"] assert plan.apply().ok @@ -404,16 +459,16 @@ def test_local_fallback_module_scope_and_ambiguity( server.nodes["device1"]["modules"].update(second["node"]["modules"]) server.vertex_forms.update(second["vertex_forms"]) with pytest.raises(ProvisioningTargetError, match="multiple ports"): - engine.plan(_device(_connection()), _blueprint()) + engine.plan(_device(_edge()), _blueprint()) blueprint = _blueprint( port_mapping={"uplink": [{"factory_label": "missing"}, {"factory_label": "P{module.position}", "kind": "ip"}]} ) device = _device( - _connection(), module_position="1", topology=ModuleTarget(device_id="device1", module_id="device1.dev.0") + _edge(), module_position="1", topology=ModuleTarget(device_id="device1", module_id="device1.dev.0") ) - assert engine.apply(device, blueprint).connections[0].local.port_id == "device1.dev.0.P1" - wrong = _connection(local=PortSelector(port_id="device1.dev.1.P1")) - with pytest.raises(ProvisioningTargetError, match="no match"): + assert engine.apply(device, blueprint).edges[0].local.port_id == "device1.dev.0.P1" + wrong = _edge(local=PortSelector(port_id="device1.dev.1.P1")) + with pytest.raises(ProvisioningTargetError, match=r"Port 'edges\.0\.local' has no match"): engine.plan(_device(wrong, topology=device.topology), _blueprint(port_mapping={})) @@ -424,10 +479,10 @@ def __getattr__(self, key: str) -> Any: engine = ProvisioningEngine(NoIO()) with pytest.raises(ProvisioningValidationError, match="Unknown local port"): - engine.plan(_device(_connection(local="unknown")), _blueprint()) + engine.plan(_device(_edge(local="unknown")), _blueprint()) with pytest.raises(ProvisioningValidationError, match="require a topology section"): engine.plan( - _device(_connection()), + _device(_edge()), Blueprint.from_dict( {"schema_version": 1, "defaults": {"inventory": {"driver_id": "com.nevion.NMOS_multidevice-0.1.0"}}} ), @@ -436,7 +491,7 @@ def __getattr__(self, key: str) -> Any: _blueprint(ip_vertex_mapping={"uplink": ["P1"]}) -def test_inventory_scope_skips_connections( +def test_inventory_scope_skips_edges( engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, @@ -445,6 +500,6 @@ def test_inventory_scope_skips_connections( blueprint = Blueprint.from_dict( {"schema_version": 1, "defaults": {"inventory": {"driver_id": "com.nevion.NMOS_multidevice-0.1.0"}}} ) - plan = engine.plan(_device(_connection()), blueprint, scope="inventory") - assert not server.calls and not plan.connections and plan.fully_resolved + plan = engine.plan(_device(_edge()), blueprint, scope="inventory") + assert not server.calls and not plan.edges and plan.fully_resolved assert plan.skipped_sections["topology"] == "not in scope 'inventory'" diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py index 51b0931..b38fad3 100644 --- a/tests/test_test_runner.py +++ b/tests/test_test_runner.py @@ -17,11 +17,11 @@ @pytest.mark.parametrize( ("selection", "expected"), [ - ([], {"test_provision", "test_connection", "test_other"}), - (["tests/e2e/provisioning"], {"test_provision", "test_connection"}), - (["tests/e2e/provisioning/test_scenario.py"], {"test_provision", "test_connection"}), + ([], {"test_provision", "test_edge", "test_other"}), + (["tests/e2e/provisioning"], {"test_provision", "test_edge"}), + (["tests/e2e/provisioning/test_scenario.py"], {"test_provision", "test_edge"}), (["tests/e2e/provisioning/test_scenario.py::test_provision"], {"test_provision"}), - (["-k", "connection"], {"test_connection"}), + (["-k", "edge"], {"test_edge"}), ], ids=["default", "directory", "file", "node", "options-only"], ) @@ -31,7 +31,7 @@ def test_e2e_entry_point_selects_only_requested_tests(tmp_path: Path, selection: scenarios.mkdir(parents=True) prefix = "import pytest\npytestmark = pytest.mark.e2e\n" (scenarios / "test_scenario.py").write_text( - prefix + "def test_provision(): pass\ndef test_connection(): pass\n", encoding="utf-8" + prefix + "def test_provision(): pass\ndef test_edge(): pass\n", encoding="utf-8" ) (scenarios.parent / "test_other.py").write_text(prefix + "def test_other(): pass\n", encoding="utf-8") (tmp_path / "tests" / "test_unit.py").write_text("def test_unit(): pass\n", encoding="utf-8") From d4a3273b4d62a60a3fcfdee0a9f4cd8c08f3f397 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:35:36 +0200 Subject: [PATCH 6/7] fix code quality findings --- tests/provisioning/test_edge_inputs.py | 2 +- tests/provisioning/test_edges.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/provisioning/test_edge_inputs.py b/tests/provisioning/test_edge_inputs.py index 8e24aa8..5b79cdd 100644 --- a/tests/provisioning/test_edge_inputs.py +++ b/tests/provisioning/test_edge_inputs.py @@ -128,7 +128,7 @@ def test_runtime_selector_errors_retain_yaml_origin_and_fail_before_io() -> None class NoIO: def __getattr__(self, key: str) -> Any: - raise AssertionError(f"unexpected I/O: {key}") + raise AttributeError(f"unexpected I/O: {key}") with pytest.raises(ProvisioningValidationError, match="exactly one") as error: ProvisioningEngine(NoIO()).plan( diff --git a/tests/provisioning/test_edges.py b/tests/provisioning/test_edges.py index 2df7e0f..5024038 100644 --- a/tests/provisioning/test_edges.py +++ b/tests/provisioning/test_edges.py @@ -475,7 +475,7 @@ def test_local_fallback_module_scope_and_ambiguity( def test_invalid_input_scope_and_mapping_keys_fail_before_io() -> None: class NoIO: def __getattr__(self, key: str) -> Any: - raise AssertionError(f"unexpected I/O: {key}") + raise AttributeError(f"unexpected I/O: {key}") engine = ProvisioningEngine(NoIO()) with pytest.raises(ProvisioningValidationError, match="Unknown local port"): From 80d03a42405d0a90e38570dbe1e885f7b1e98255 Mon Sep 17 00:00:00 2001 From: Jonas Scholl <43819845+JonasScholl@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:18:28 +0200 Subject: [PATCH 7/7] review --- docs/architecture/provisioning/reference.md | 26 +- .../apps/inventory/errors.py | 26 +- .../apps/inventory/inventory_api.py | 39 ++- .../provisioning/edges.py | 8 +- .../provisioning/engine.py | 25 +- .../provisioning/errors.py | 5 + .../provisioning/inspect.py | 16 +- .../provisioning/models.py | 8 +- .../provisioning/naming.py | 18 +- tests/inventory/test_inventory_api.py | 45 +++ tests/provisioning/test_edges.py | 55 +++- tests/provisioning/test_engine.py | 25 +- tests/provisioning/test_plan_guards.py | 257 ++++++++++++++++++ 13 files changed, 512 insertions(+), 41 deletions(-) create mode 100644 tests/provisioning/test_plan_guards.py diff --git a/docs/architecture/provisioning/reference.md b/docs/architecture/provisioning/reference.md index ed4c971..b6482b4 100644 --- a/docs/architecture/provisioning/reference.md +++ b/docs/architecture/provisioning/reference.md @@ -65,7 +65,7 @@ replay elsewhere. | Field | Meaning | |---|---| | `status` | `succeeded`, `no_change`, `planned` (dry run), `failed`, `partial`, or `unknown`. `partial` with `replan_required` is returned, not raised. | -| `ok` | `True` for `succeeded`, `no_change`, and `planned`. | +| `ok` | `True` for `succeeded`, `no_change`, and `planned`, except when `verification` is `unconfirmed`. A succeeded write with an unconfirmed read-back stays `succeeded`; `ok` is false so it is not treated as confirmed. | | `replan_required` | `True` after an Inventory update requiring rediscovery, or when peer edges remain open. Real runs return `partial`; dry runs return `planned`. Create a new plan when discovery is ready. | | `inventory_id`, `topology_device_id`, `module_id`, `source_key` | Ids involved. Persist `inventory_id` in your source system. | | `phases`, `phase(name)` | `PhaseResult` per phase: `completed`, `no_change`, `skipped`, `failed`, `unknown`, `not_run`, `planned`, or `deferred`. `deferred` reports work still waiting for discovery or a new plan. | @@ -537,11 +537,16 @@ default, then the engine `naming`, then the blueprint (a topology entry replaces the same inventory entry), then the `naming` argument on that call. An entry is replaced as a whole; expressions are not merged. Set an entry to `None` to turn that name off. `NamingScheme.layered(*schemes)` performs the same combination. +Declarative expressions are copied recursively when configuring engine defaults +and capturing a plan, including nested `Join.parts` and `Field.mapping` values. +Later mutations of caller expressions or engine defaults do not change an +existing plan's naming, including work deferred until discovery. When the blocks cannot express a name, put a computed value in `attributes`, or pass a Python object with `render(context: NameContext) -> str` (the `NameRenderer` protocol) for that entry. Python renderers are not available in -YAML. +YAML. These trusted callbacks are retained by reference: their state and behavior +remain caller-controlled, so keep them stable while a saved plan is pending. **Checks.** Before any write, every rendered endpoint label is checked. It has to be non-empty, it cannot contain control characters, and it cannot duplicate @@ -679,9 +684,19 @@ needs a sync, the plan reports a diagnostic. Before writing, the engine reads the Inventory record and the scoped topology again. If anything the plan relied on has changed, it raises -`ProvisioningConflictError`. An Inventory update also repeats the label and address +`ProvisioningConflictError`. Inventory baseline checks also run when the plan +requires no Inventory update; the fresh record remains available to deferred +processors. An Inventory update also repeats the label and address conflict check. A response with no address item list fails that check. Build a -new plan. The Inspect transaction then checks +new plan. + +For fully resolved device topology plans using `add_only` or `reconcile`, apply +also reads synchronization status again. Pending synchronization invalidates the +plan and requires a new one; apply does not synchronize and replace the reviewed +work. Lookup failures propagate. `sync="none"` and module targets retain their +policy of working against the current topology without synchronization. + +The Inspect transaction then checks its own baseline, and module tags are read again immediately before their phase. These checks run in the client, so a short gap between the read and the write remains. Uncommitted edits on `app.inspect` that overlap the plan are rejected, @@ -721,7 +736,8 @@ What you do next depends on how far the apply got: - The record was created, then discovery timed out. `status` is `partial` and `inventory_id` is set. Retry with `ProvisioningDevice(..., inventory_id=result.inventory_id)`. - Topology committed, then a module tag failed. `status` is `partial`. The result lists the committed edits and the tag operations that finished. Replan to finish the rest. - A write timed out. `status` is `unknown`. Read the server back before you retry. Building a new plan does that read. -- Verification could not confirm the write, or a later phase failed before the topology read-back ran. The write stays applied, and `verification` is `"unconfirmed"`. +- The record was inserted and tracking-id cleanup did not finish. `status` is `unknown` and `inventory_id` is set. Bind that id; do not create again. +- Verification could not confirm the write, or a later phase failed before the topology read-back ran. The write stays applied, `verification` is `"unconfirmed"`, and `ok` is false. `status` stays `succeeded` when the write itself completed. Read the server back before retrying. The engine does not roll a partial apply back. diff --git a/src/videoipath_automation_tool/apps/inventory/errors.py b/src/videoipath_automation_tool/apps/inventory/errors.py index 5a0def7..38c835e 100644 --- a/src/videoipath_automation_tool/apps/inventory/errors.py +++ b/src/videoipath_automation_tool/apps/inventory/errors.py @@ -14,13 +14,31 @@ class InventoryStatusUnavailableError(ValueError): class InventoryWriteNotAppliedError(ValueError): """An Inventory write was rejected by the server or not attempted, so it did not change the record. - ``operation`` names the write that was not applied. Subclasses ``ValueError`` so existing callers - that catch ``ValueError`` keep working. + ``operation`` names the write that was not applied. ``device_id`` is set when the record is already + known, including a tracking-id cleanup that runs after a successful add. Subclasses ``ValueError`` + so existing callers that catch ``ValueError`` keep working. """ - def __init__(self, message: str, *, operation: InventoryWriteOperation) -> None: + def __init__(self, message: str, *, operation: InventoryWriteOperation, device_id: str | None = None) -> None: self.operation: InventoryWriteOperation = operation + self.device_id = device_id super().__init__(message) -__all__ = ["InventoryStatusUnavailableError", "InventoryWriteNotAppliedError", "InventoryWriteOperation"] +class InventoryRecordCreatedError(Exception): + """The Inventory record exists, and a later step in ``add_device`` did not finish. + + ``device_id`` is the new record. The unfinished step's outcome is not known. + """ + + def __init__(self, device_id: str, message: str) -> None: + self.device_id = device_id + super().__init__(message) + + +__all__ = [ + "InventoryRecordCreatedError", + "InventoryStatusUnavailableError", + "InventoryWriteNotAppliedError", + "InventoryWriteOperation", +] diff --git a/src/videoipath_automation_tool/apps/inventory/inventory_api.py b/src/videoipath_automation_tool/apps/inventory/inventory_api.py index 5604ba9..3980ac1 100644 --- a/src/videoipath_automation_tool/apps/inventory/inventory_api.py +++ b/src/videoipath_automation_tool/apps/inventory/inventory_api.py @@ -9,6 +9,7 @@ from typing_extensions import deprecated from videoipath_automation_tool.apps.inventory.errors import ( + InventoryRecordCreatedError, InventoryStatusUnavailableError, InventoryWriteNotAppliedError, ) @@ -193,9 +194,17 @@ def add_device( self._logger.debug( "Remove tracking ID ('uuid' meta field) from device configuration and update device in VideoIPath-Inventory." ) - online_device = self.update_device( - device=modified_device, config_only=True - ) # Use config_only=True to avoid status fetch and speed up the process + try: + online_device = self.update_device(device=modified_device, config_only=True) + except InventoryWriteNotAppliedError as exc: + raise InventoryWriteNotAppliedError( + str(exc), operation=exc.operation, device_id=online_device.configuration.id + ) from exc + except Exception as exc: + raise InventoryRecordCreatedError( + online_device.configuration.id, + f"Tracking-id cleanup failed for '{online_device.configuration.id}': {type(exc).__name__}: {exc}", + ) from exc self._logger.debug("Tracking ID removed successfully from device configuration.") if config_only: @@ -262,7 +271,9 @@ def update_device( if response.header.status != "OK": raise InventoryWriteNotAppliedError( - f"Failed to update device in VideoIPath-Inventory. Error: {response}", operation="update" + f"Failed to update device in VideoIPath-Inventory. Error: {response}", + operation="update", + device_id=device_id, ) online_device = self.get_device( @@ -570,18 +581,20 @@ def get_device_id_by_address(self, address: str, include_alt_addresses: bool = T response = self.vip_connector.rest.get(url) - if response.data and isinstance(response.data["config"]["devman"]["devices"]["_items"], list): - devices = response.data["config"]["devman"]["devices"]["_items"] - else: + devices = ( + response.data.get("config", {}).get("devman", {}).get("devices", {}).get("_items") + if isinstance(response.data, dict) + else None + ) + if not isinstance(devices, list): raise ValueError("Response data is empty.") - device_ids = [] for device in devices: - if ( - address in device["config"]["cinfo"].get("altAddresses") - or address == device["config"]["cinfo"]["address"] - ): - device_ids.append(device["_id"]) + if not isinstance(device, dict): + continue + found_id = device.get("_id") + if isinstance(found_id, str): + device_ids.append(found_id) if len(device_ids) == 0: return None diff --git a/src/videoipath_automation_tool/provisioning/edges.py b/src/videoipath_automation_tool/provisioning/edges.py index e245d3b..a0c013f 100644 --- a/src/videoipath_automation_tool/provisioning/edges.py +++ b/src/videoipath_automation_tool/provisioning/edges.py @@ -12,6 +12,7 @@ ProvisioningTargetError, ProvisioningValidationError, TopologyNotReadyError, + UndirectedPortError, ) from videoipath_automation_tool.provisioning.models import ( EdgePatch, @@ -103,7 +104,7 @@ def resolve_port(scope: ScopeData, candidates: list[PortSelector], *, key: str) raise ProvisioningTargetError(f"Port '{port.id}' has multiple '{direction}' vertices.") directions[direction] = matches[0] if matches else None if not any(directions.values()): - raise ProvisioningTargetError(f"Port '{port.id}' has no directed In/Out vertex.") + raise UndirectedPortError(f"Port '{port.id}' has no directed In/Out vertex.") return PortBinding( key=key, candidate=candidate.port_id or candidate.factory_label or "", @@ -143,7 +144,10 @@ def resolve_edges( edge=edge, local=local, status="deferred", - reason="Peer device, module or ports are not discovered in topology; create a new plan when available.", + reason=( + "Peer device, module, ports, or directed vertices are not available in topology; " + "create a new plan when available." + ), ) ) continue diff --git a/src/videoipath_automation_tool/provisioning/engine.py b/src/videoipath_automation_tool/provisioning/engine.py index 8723c6f..6ffcbf8 100644 --- a/src/videoipath_automation_tool/provisioning/engine.py +++ b/src/videoipath_automation_tool/provisioning/engine.py @@ -280,7 +280,7 @@ def configure( if options is not _UNSET: self._options = options.model_copy() if options is not None else ApplyOptions() if naming is not _UNSET: - self._naming = naming.model_copy() if naming is not None else None + self._naming = NamingScheme.layered(naming) if naming is not None else None def validate( self, @@ -764,9 +764,6 @@ def _run_inventory(self) -> None: return phase = self._enter("inventory") gateway = self._inventory() - if work.action == "none": - phase.status = "no_change" - return if work.action == "create": recheck_conflicts(gateway, self._captured.device, work) candidate = build_candidate(new_device(work.driver_id), work.desired) @@ -774,12 +771,22 @@ def _run_inventory(self) -> None: self._planned(phase, [work.operation]) return with self._write(create=True): - online = gateway.create(candidate) + try: + online = gateway.create(candidate) + except Exception as exc: + created_id = getattr(exc, "device_id", None) + if isinstance(created_id, str): + self._result.inventory_id = created_id + raise self._result.inventory_id = online.device_id else: assert work.inventory_id is not None fresh = gateway.read(work.inventory_id) check_baseline(work, fresh) + self._current_inventory = fresh + if work.action == "none": + phase.status = "no_change" + return recheck_conflicts(gateway, self._captured.device, work) if self._dry_run: self._planned(phase, [work.operation]) @@ -834,6 +841,14 @@ def _run_topology(self) -> None: assert captured.topology_work is not None work = captured.topology_work self._enter("topology") + if ( + not isinstance(target, ModuleTarget) + and captured.options.sync != "none" + and _sync_pending(gateway.sync_info(target.device_id)) + ): + raise ProvisioningConflictError( + f"Topology synchronization for '{target.device_id}' is pending; create a new plan." + ) if gateway.read_scope(target).fingerprint != work.fingerprint: raise ProvisioningConflictError( f"The topology of {_target_label(target)} changed since planning; create a new plan." diff --git a/src/videoipath_automation_tool/provisioning/errors.py b/src/videoipath_automation_tool/provisioning/errors.py index ea08724..eb3c8de 100644 --- a/src/videoipath_automation_tool/provisioning/errors.py +++ b/src/videoipath_automation_tool/provisioning/errors.py @@ -66,6 +66,10 @@ class ProvisioningTargetError(ProvisioningError): """A binding is missing or ambiguous, or a target lies outside the requested scope.""" +class UndirectedPortError(ProvisioningTargetError): + """A matched port has no directed In/Out vertex yet.""" + + class ProcessorInputError(ProvisioningError): """A processor cannot interpret the observed topology (unsupported or incomplete layout).""" @@ -113,5 +117,6 @@ def __init__(self, result: ApplyResult, message: str) -> None: "ProvisioningTargetError", "ProvisioningValidationError", "TopologyNotReadyError", + "UndirectedPortError", "ValidationIssue", ] diff --git a/src/videoipath_automation_tool/provisioning/inspect.py b/src/videoipath_automation_tool/provisioning/inspect.py index 2646257..2ef6f2c 100644 --- a/src/videoipath_automation_tool/provisioning/inspect.py +++ b/src/videoipath_automation_tool/provisioning/inspect.py @@ -44,6 +44,7 @@ ProvisioningTargetError, ProvisioningValidationError, TopologyNotReadyError, + UndirectedPortError, ValidationIssue, ) from videoipath_automation_tool.provisioning.models import ( @@ -309,8 +310,12 @@ def read_edge_peers(self, edges: list[ProvisioningEdge]) -> dict[int, ScopeData if key not in cache: cache[key] = self.read_peer_scope(target) peers[index] = cache[key] - if peers[index] is not None: - resolve_port(peers[index], [edge.peer.port], key=f"edges.{index}.peer") + peer_scope = peers[index] + if peer_scope is not None: + try: + resolve_port(peer_scope, [edge.peer.port], key=f"edges.{index}.peer") + except UndirectedPortError: + peers[index] = None return peers def check_peers(self, edges: list[ProvisioningEdge], peers: dict[int, ScopeData | None]) -> None: @@ -402,7 +407,12 @@ def commit(self, work: TopologyWork) -> Any: if edge.operation is None: continue if edge.baseline is None: - tx.connect(edge.from_vertex, edge.to_vertex, bidirectional=False, **edge.intents) + try: + tx.connect(edge.from_vertex, edge.to_vertex, bidirectional=False, **edge.intents) + except ValueError as exc: + if "already exists" not in str(exc): + raise + raise ProvisioningConflictError(str(exc)) from exc else: tx.update_edge(edge_id, intents=dict(edge.intents)) self.check_edges(work.edge_work) diff --git a/src/videoipath_automation_tool/provisioning/models.py b/src/videoipath_automation_tool/provisioning/models.py index e1ac0d2..6b6a0c2 100644 --- a/src/videoipath_automation_tool/provisioning/models.py +++ b/src/videoipath_automation_tool/provisioning/models.py @@ -690,7 +690,13 @@ class ApplyResult(BaseModel): @property def ok(self) -> bool: - return self.status in ("succeeded", "no_change", "planned") + """True when status is a finished non-failure and read-back did not disagree. + + ``succeeded`` with ``verification == "unconfirmed"`` is not ok. ``status`` stays + ``succeeded`` because the write was applied; check ``verification`` before treating + the outcome as confirmed. + """ + return self.verification != "unconfirmed" and self.status in ("succeeded", "no_change", "planned") def phase(self, name: PhaseName) -> PhaseResult | None: return next((phase for phase in self.phases if phase.name == name), None) diff --git a/src/videoipath_automation_tool/provisioning/naming.py b/src/videoipath_automation_tool/provisioning/naming.py index 4f7cc1b..3e50cff 100644 --- a/src/videoipath_automation_tool/provisioning/naming.py +++ b/src/videoipath_automation_tool/provisioning/naming.py @@ -119,7 +119,10 @@ def value(self, path: str) -> Any: @runtime_checkable class NameRenderer(Protocol): - """Trusted Python escape hatch for one naming entry; never loaded from YAML.""" + """Trusted Python escape hatch for one naming entry; never loaded from YAML. + + Renderers are retained by reference; their state and behavior remain caller-controlled. + """ def render(self, context: NameContext) -> str: """Return the name for ``context``.""" @@ -162,12 +165,21 @@ def entries(self) -> dict[str, NamingEntry | None]: @classmethod def layered(cls, *schemes: NamingScheme | None) -> NamingScheme: - """Combine schemes entry by entry; later schemes win for the entries they supply.""" + """Combine schemes entry by entry; later schemes win for the entries they supply. + + Declarative expressions are deep-copied, including nested lists and mappings. Trusted + Python renderers are retained by reference. + """ merged: dict[str, Any] = {} for scheme in schemes: if scheme is not None: merged.update(scheme.entries()) - return cls(**merged) + return cls( + **{ + name: expression.model_copy(deep=True) if isinstance(expression, (Text, Field, Join)) else expression + for name, expression in merged.items() + } + ) class BlueprintNaming(BaseModel): diff --git a/tests/inventory/test_inventory_api.py b/tests/inventory/test_inventory_api.py index 3933bd4..d6be401 100644 --- a/tests/inventory/test_inventory_api.py +++ b/tests/inventory/test_inventory_api.py @@ -10,6 +10,7 @@ from videoipath_automation_tool.apps.inventory.app.app import InventoryApp from videoipath_automation_tool.apps.inventory.errors import ( + InventoryRecordCreatedError, InventoryStatusUnavailableError, InventoryWriteNotAppliedError, ) @@ -122,6 +123,16 @@ def test_get_device_id_by_address_keeps_exact_matching() -> None: _api(None).get_device_id_by_address("192.0.2.1") +def test_get_device_id_by_address_without_alternates_reads_id_projection() -> None: + projected = {"config": {"devman": {"devices": {"_items": [{"_id": "device1"}]}}}} + api = _api(projected) + assert api.get_device_id_by_address("192.0.2.1", include_alt_addresses=False) == "device1" + assert api.vip_connector.rest.get_calls[0].endswith("/_id") + assert _api(_devices()).get_device_id_by_address("192.0.2.1", include_alt_addresses=False) is None + with pytest.raises(ValueError, match="Response data is empty"): + _api(None).get_device_id_by_address("192.0.2.1", include_alt_addresses=False) + + # --- Write rejections --- @@ -138,6 +149,40 @@ def test_rejected_update_is_typed() -> None: with pytest.raises(InventoryWriteNotAppliedError, match="Failed to update device") as info: _api(rpc_status="ERROR").update_device(device, config_only=True) assert info.value.operation == "update" + assert info.value.device_id == "device1" + + +def test_rejected_tracking_cleanup_keeps_the_created_id(monkeypatch: pytest.MonkeyPatch) -> None: + api = _api() + created = InventoryDevice.create(NMOS) + created.configuration.id = "device9" + created.configuration.meta["uuid"] = "tracking-id" + monkeypatch.setattr(api, "_fetch_device_config_by_uuid", lambda uuid: created) + + def reject(*args: Any, **kwargs: Any) -> None: + raise InventoryWriteNotAppliedError("rejected", operation="update") + + monkeypatch.setattr(api, "update_device", reject) + with pytest.raises(InventoryWriteNotAppliedError, match="rejected") as info: + api.add_device(InventoryDevice.create(NMOS), config_only=True) + assert info.value.operation == "update" + assert info.value.device_id == "device9" + + +def test_tracking_cleanup_timeout_reports_the_created_id(monkeypatch: pytest.MonkeyPatch) -> None: + api = _api() + created = InventoryDevice.create(NMOS) + created.configuration.id = "device9" + created.configuration.meta["uuid"] = "tracking-id" + monkeypatch.setattr(api, "_fetch_device_config_by_uuid", lambda uuid: created) + + def time_out(*args: Any, **kwargs: Any) -> None: + raise TimeoutError("timed out") + + monkeypatch.setattr(api, "update_device", time_out) + with pytest.raises(InventoryRecordCreatedError, match="device9") as info: + api.add_device(InventoryDevice.create(NMOS), config_only=True) + assert info.value.device_id == "device9" def test_failed_preread_before_update_is_typed(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/provisioning/test_edges.py b/tests/provisioning/test_edges.py index 5024038..dbd427d 100644 --- a/tests/provisioning/test_edges.py +++ b/tests/provisioning/test_edges.py @@ -24,6 +24,7 @@ PeerEndpoint, PortSelector, ProvisioningApplyError, + ProvisioningConflictError, ProvisioningDevice, ProvisioningEdge, ProvisioningEngine, @@ -32,6 +33,7 @@ TagDelta, models, ) +from videoipath_automation_tool.provisioning.errors import UndirectedPortError def _blueprint(**topology: Any) -> Blueprint: @@ -305,6 +307,57 @@ def fail(device_id: str) -> None: assert not server.writes +def test_undirected_peer_port_defers_that_edge( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _seed(inventory, server) + server.nodes["device2"]["modules"]["device2.dev.0"]["ports"]["device2.dev.0.P1"]["vertexInfo"] = {"type": "double"} + peer = PeerEndpoint(target=DeviceTarget(device_id="device2"), port=PortSelector(factory_label="P2")) + blueprint = _blueprint(port_mapping={"uplink": [{"factory_label": "P1"}], "backup": [{"factory_label": "P2"}]}) + plan = engine.plan( + _device(_edge(), _edge(local="backup", peer=peer)), + blueprint, + ) + assert [state.status for state in plan.edges] == ["deferred", "planned"] + assert not server.writes + + server.nodes["device1"]["modules"]["device1.dev.0"]["ports"]["device1.dev.0.P1"]["vertexInfo"] = {"type": "double"} + with pytest.raises(UndirectedPortError, match="no directed"): + engine.plan(_device(_edge()), _blueprint()) + assert not server.writes + + +def test_edge_appearing_before_connect_is_a_conflict( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _seed(inventory, server) + plan = engine.plan(_device(_edge()), _blueprint()) + lookups = {"count": 0} + original = server.lookup_edges + + def lookup(edge_ids: list[str]) -> Any: + lookups["count"] += 1 + if lookups["count"] == 1: + result = original(edge_ids) + for edge_id in edge_ids: + from_id, _, to_id = edge_id.partition("::") + server.edge_forms[edge_id] = InspectApiEdgeForm(fromId=from_id, toId=to_id).model_dump() + return result + return original(edge_ids) + + monkeypatch.setattr(server, "lookup_edges", lookup) + with pytest.raises(ProvisioningApplyError) as info: + plan.apply() + result = info.value.result + assert result.status == "failed" + assert result.phase("topology").status == "failed" + assert isinstance(info.value.__cause__, ProvisioningConflictError) + assert not any(name == "update_topology" for name, _ in server.writes) + + @pytest.mark.parametrize("change", ["peer", "edge_created", "edge_updated", "edge_removed"]) @pytest.mark.parametrize("dry_run", [False, True]) def test_stale_peer_and_edge_baselines( @@ -422,7 +475,7 @@ def discard_edges(delta: Any) -> Any: monkeypatch.setattr(server, "update_topology", discard_edges) result = engine.apply(_device(_edge()), _blueprint()) - assert result.status == "succeeded" and result.verification == "unconfirmed" + assert result.status == "succeeded" and result.verification == "unconfirmed" and not result.ok assert "fromId" in result.verification_detail diff --git a/tests/provisioning/test_engine.py b/tests/provisioning/test_engine.py index 73a4d7d..183ca74 100644 --- a/tests/provisioning/test_engine.py +++ b/tests/provisioning/test_engine.py @@ -13,7 +13,10 @@ from tests.provisioning.conftest import NMOS, FakeInspectServer, FakeInventory, make_inspect_app, matrox_layout from videoipath_automation_tool.apps.inspect.snapshot import InspectSnapshot -from videoipath_automation_tool.apps.inventory.errors import InventoryWriteNotAppliedError +from videoipath_automation_tool.apps.inventory.errors import ( + InventoryRecordCreatedError, + InventoryWriteNotAppliedError, +) from videoipath_automation_tool.provisioning import ( AlternativeAddress, ApplyOptions, @@ -655,16 +658,30 @@ def test_inventory_preread_failure_is_a_known_rejection( assert inventory.writes == [] -@pytest.mark.parametrize(("operation", "status"), [("add", "failed"), ("update", "unknown")]) +@pytest.mark.parametrize( + ("operation", "status", "device_id"), + [("add", "failed", None), ("update", "unknown", "device9")], +) def test_inventory_create_rejection_outcomes( - engine: ProvisioningEngine, inventory: FakeInventory, operation: str, status: str + engine: ProvisioningEngine, inventory: FakeInventory, operation: str, status: str, device_id: str | None ) -> None: # A rejected add is known; a rejected follow-up update means the record may already exist. - inventory.fail_next_write = InventoryWriteNotAppliedError("rejected", operation=operation) # type: ignore[arg-type] + inventory.fail_next_write = InventoryWriteNotAppliedError("rejected", operation=operation, device_id=device_id) # type: ignore[arg-type] device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") with pytest.raises(ProvisioningApplyError) as info: engine.apply(device, _blueprint(), scope="inventory") assert info.value.result.status == status + assert info.value.result.inventory_id == device_id + + +def test_create_cleanup_timeout_reports_the_new_id(engine: ProvisioningEngine, inventory: FakeInventory) -> None: + inventory.fail_next_write = InventoryRecordCreatedError("device9", "Tracking-id cleanup failed: timed out") + device = ProvisioningDevice(key="key-new", label="device-new", management_address="192.0.2.50") + with pytest.raises(ProvisioningApplyError) as info: + engine.apply(device, _blueprint(), scope="inventory") + result = info.value.result + assert result.status == "unknown" + assert result.inventory_id == "device9" def test_untyped_inventory_value_error_is_an_unknown_outcome( diff --git a/tests/provisioning/test_plan_guards.py b/tests/provisioning/test_plan_guards.py new file mode 100644 index 0000000..e833845 --- /dev/null +++ b/tests/provisioning/test_plan_guards.py @@ -0,0 +1,257 @@ +"""Regression coverage for captured Inventory, synchronization, and naming state.""" + +from __future__ import annotations + +from typing import Any, Literal + +import pytest +from pydantic import BaseModel, ConfigDict + +from tests.provisioning.conftest import NMOS, FakeInspectServer, FakeInventory, matrox_layout +from tests.provisioning.test_engine import _blueprint, _device, _existing +from videoipath_automation_tool.provisioning import ( + ApplyOptions, + Blueprint, + Field, + Join, + ModuleTarget, + NameContext, + NamingScheme, + ProcessingContext, + ProcessorResult, + ProvisioningApplyError, + ProvisioningConflictError, + ProvisioningDevice, + ProvisioningEngine, + ProvisioningError, + TopologyNotReadyError, + VertexEdit, + VertexPatch, + VertexProcessor, +) + + +@pytest.mark.parametrize("dry_run", [False, True]) +@pytest.mark.parametrize("changed", ["managed_setting", "driver"]) +def test_unchanged_inventory_rechecks_baseline( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + dry_run: bool, + changed: str, +) -> None: + _existing(inventory, server) + plan = engine.plan(_device(), _blueprint()) + assert plan.phase("inventory").status == "no_change" + if changed == "managed_setting": + inventory.devices["device1"].configuration.config.customSettings.port = 9090 + else: + inventory.devices["device1"].configuration.config.driver.version = "0.2.0" + + with pytest.raises(ProvisioningApplyError, match="changed since planning") as caught: + plan.apply(dry_run=dry_run) + assert isinstance(caught.value.__cause__, ProvisioningConflictError) + assert ("driver_id" if changed == "driver" else "custom_settings.port") in str(caught.value) + assert caught.value.result.phase("inventory").status == "failed" + assert inventory.writes == [] and server.writes == [] + + +@pytest.mark.parametrize("dry_run", [False, True]) +def test_unchanged_inventory_allows_unmanaged_changes( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, dry_run: bool +) -> None: + _existing(inventory, server) + plan = engine.plan(_device(), _blueprint()) + inventory.devices["device1"].configuration.metadata["note"] = "updated externally" + + result = plan.apply(dry_run=dry_run) + assert result.status == ("planned" if dry_run else "succeeded") + assert result.phase("inventory").status == "no_change" and inventory.writes == [] + assert inventory.devices["device1"].configuration.metadata["note"] == "updated externally" + + +class _ContextParams(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class _InventoryContextProcessor(VertexProcessor[_ContextParams]): + params_model = _ContextParams + + def process(self, context: ProcessingContext, params: _ContextParams) -> ProcessorResult: + assert context.inventory is not None and context.inventory.inventory_id == "device5" + assert context.owner is not None and context.owner.inventory_id == "device1" + vertex = context.require_vertex(factory_label="Video Sender") + if vertex.kind is None: + raise TopologyNotReadyError("Waiting for the vertex edit form.") + return ProcessorResult( + vertices=[VertexEdit(vertex_id=vertex.id, fields=VertexPatch(label=context.inventory.inventory_id))] + ) + + +@pytest.mark.parametrize("require_reachable", [False, True]) +def test_deferred_processor_keeps_unchanged_inventory_context( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, require_reachable: bool +) -> None: + _existing(inventory, server) + inventory.seed("device5", label="module-a", address="192.0.2.55") + engine.register_processor("example-org.inventory-context", _InventoryContextProcessor) + device = ProvisioningDevice( + key="module-a", + label="module-a", + inventory_id="device5", + topology=ModuleTarget(device_id="device1", module_id="device1.dev.0"), + ) + blueprint = Blueprint.from_dict( + { + "schema_version": 1, + "defaults": { + "inventory": {"driver_id": NMOS}, + "topology": {"vertex_processor": {"processor_type": "example-org.inventory-context"}}, + }, + } + ) + form = server.vertex_forms.pop("device1.0.vs.v") + plan = engine.plan(device, blueprint, options=ApplyOptions(require_reachable=require_reachable)) + assert not plan.fully_resolved and plan.phase("inventory").status == "no_change" + server.vertex_forms["device1.0.vs.v"] = form + + result = plan.apply() + assert result.status == "succeeded" and result.materialized + assert result.phase("inventory").status == "no_change" and inventory.writes == [] + assert inventory.status_reads == (["device5"] if require_reachable else []) + assert server.vertex_forms["device1.0.vs.v"]["fields"]["label"] == "device5" + + +@pytest.mark.parametrize("dry_run", [False, True]) +@pytest.mark.parametrize("sync", ["add_only", "reconcile"]) +@pytest.mark.parametrize("pending", ["add", "update", "remove"]) +def test_resolved_plan_rejects_new_pending_sync( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + dry_run: bool, + sync: Literal["add_only", "reconcile"], + pending: str, +) -> None: + _existing(inventory, server) + plan = engine.plan(_device(), _blueprint(), options=ApplyOptions(sync=sync)) + assert plan.fully_resolved + server.sync["device1"] = {"add": {}, "update": {}, "remove": {}, pending: {"vertices": 1}} + + with pytest.raises(ProvisioningApplyError, match="synchronization.*new plan") as caught: + plan.apply(dry_run=dry_run) + assert isinstance(caught.value.__cause__, ProvisioningConflictError) + assert caught.value.result.status == "failed" + assert inventory.writes == [] and server.writes == [] + + +@pytest.mark.parametrize("dry_run", [False, True]) +def test_resolved_plan_propagates_sync_lookup_failure( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer, dry_run: bool +) -> None: + _existing(inventory, server) + plan = engine.plan(_device(), _blueprint()) + server.fail_sync_info = ConnectionError("sync lookup failed") + + with pytest.raises(ProvisioningApplyError, match="synchronization status") as caught: + plan.apply(dry_run=dry_run) + assert type(caught.value.__cause__) is ProvisioningError + assert inventory.writes == [] and server.writes == [] + + +def test_sync_conflict_retains_completed_inventory_phase( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + _existing(inventory, server) + inventory.devices["device1"].configuration.label = "old-label" + plan = engine.plan(_device(), _blueprint()) + assert plan.fully_resolved + server.sync["device1"] = {"add": {"vertices": 1}, "update": {}, "remove": {}} + + with pytest.raises(ProvisioningApplyError) as caught: + plan.apply() + result = caught.value.result + assert isinstance(caught.value.__cause__, ProvisioningConflictError) + assert result.status == "partial" and result.inventory_id == "device1" + assert result.phase("inventory").status == "completed" + assert result.phase("inventory").operations + assert inventory.writes == [("update", "device1")] and server.writes == [] + + +@pytest.mark.parametrize( + ("target_kind", "sync"), + [("device", "none"), ("module", "none"), ("module", "add_only"), ("module", "reconcile")], +) +def test_resolved_plan_preserves_explicit_no_sync_policies( + engine: ProvisioningEngine, + inventory: FakeInventory, + server: FakeInspectServer, + target_kind: str, + sync: Literal["none", "add_only", "reconcile"], +) -> None: + _existing(inventory, server) + target = ModuleTarget(device_id="device1", module_id="device1.dev.0") if target_kind == "module" else None + plan = engine.plan(_device(topology=target), _blueprint(device=None), options=ApplyOptions(sync=sync)) + server.sync["device1"] = {"add": {}, "update": {"vertices": 1}, "remove": {}} + + result = plan.apply() + assert result.status == "succeeded" + assert not any(name in {"add_devices", "sync_devices"} for name, _ in server.writes) + + +@pytest.mark.parametrize("source", ["engine_before_plan", "engine_after_plan", "call"]) +@pytest.mark.parametrize("mutation", ["parts", "mapping"]) +def test_deferred_plan_captures_nested_naming( + engine: ProvisioningEngine, server: FakeInspectServer, source: str, mutation: str +) -> None: + media = Field("endpoint.media", mapping={"video": "video", "audio": "audio"}) + nested = Join(parts=[media, Field("endpoint.index", format="02d")], separator="-") + naming = NamingScheme(endpoint_label=Join(parts=[Field("device.label"), nested], separator="-")) + device = ProvisioningDevice(key="device-a", label="device-a", management_address="192.0.2.10") + server.discoverable["device100"] = matrox_layout("device100") + if source != "call": + engine.configure(naming=naming) + + def mutate() -> None: + if mutation == "parts": + nested.parts.insert(0, "later") + else: + media.mapping["video"] = "later" + + if source == "engine_before_plan": + mutate() + plan = engine.plan(device, _blueprint(), naming=naming if source == "call" else None) + assert not plan.fully_resolved + if source == "call": + mutate() + elif source == "engine_after_plan": + # Engine defaults are caller-accessible too; captured plans must be independent of them. + engine_expression = engine.naming.endpoint_label + if mutation == "parts": + engine_expression.parts[1].parts.insert(0, "later") + else: + engine_expression.parts[1].parts[0].mapping["video"] = "later" + + result = plan.apply() + assert result.status == "succeeded" + assert server.vertex_forms["device100.0.vs.v"]["fields"]["label"] == "device-a-video-01" + assert server.vertex_forms["device100.0.as.v"]["fields"]["label"] == "device-a-audio-01" + + +def test_captured_naming_keeps_trusted_python_renderer( + engine: ProvisioningEngine, inventory: FakeInventory, server: FakeInspectServer +) -> None: + class Renderer: + def __deepcopy__(self, memo: dict[int, Any]) -> Renderer: + raise TypeError("Trusted callback must not be copied.") + + def render(self, context: NameContext) -> str: + return "device-a-custom" + + _existing(inventory, server) + renderer = Renderer() + engine.configure(naming=NamingScheme(device_label=renderer)) + assert engine.naming.device_label is renderer + result = engine.plan(_device(), _blueprint()).apply() + assert result.status == "succeeded" + assert server.device_forms["device1"]["descriptor"]["label"] == "device-a-custom"