diff --git a/.github/workflows/schema-drift.yml b/.github/workflows/schema-drift.yml new file mode 100644 index 000000000..86e37c453 --- /dev/null +++ b/.github/workflows/schema-drift.yml @@ -0,0 +1,32 @@ +--- +# yamllint disable rule:truthy rule:line-length +name: Schema Drift Check + +# Warn-only: surfaces when the published Infrahub JSON schema has drifted from +# the formatter's committed baseline (infrahub_sdk/ctl/schema_properties.json). +# This never fails the run — it emits ::warning:: annotations and a job summary +# so a maintainer can account for the change in schema_format.py. + +on: + release: + types: + - published + workflow_dispatch: + +jobs: + schema-drift: + runs-on: "ubuntu-22.04" + timeout-minutes: 5 + steps: + - name: "Check out repository code" + uses: "actions/checkout@v6" + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install UV + uses: astral-sh/setup-uv@v7 + - name: Install dependencies + run: uv sync --all-groups --all-extras + - name: "Check for Infrahub schema drift (warn only)" + run: uv run invoke schema-drift-check diff --git a/changelog/+schema-format-command.added.md b/changelog/+schema-format-command.added.md new file mode 100644 index 000000000..7d34b7b75 --- /dev/null +++ b/changelog/+schema-format-command.added.md @@ -0,0 +1 @@ +Add `infrahubctl schema format` command, an opinionated offline formatter that normalises the key ordering of schema files. Optional flags can also strip redundant default values (`--strip-defaults`), sort attributes/relationships by `order_weight` (`--sort-by-order-weight`), and backfill a missing `order_weight` (`--backfill-order-weight`). diff --git a/docs/docs/infrahubctl/infrahubctl-schema.mdx b/docs/docs/infrahubctl/infrahubctl-schema.mdx index 2bab8a77c..6cffe318f 100644 --- a/docs/docs/infrahubctl/infrahubctl-schema.mdx +++ b/docs/docs/infrahubctl/infrahubctl-schema.mdx @@ -21,6 +21,7 @@ $ infrahubctl schema [OPTIONS] COMMAND [ARGS]... * `export`: Export the schema from Infrahub as YAML... * `list`: List all available schema kinds. * `show`: Show details for a specific schema kind. +* `format`: Format Infrahub schema files with a... ## `infrahubctl schema load` @@ -133,3 +134,45 @@ $ infrahubctl schema show [OPTIONS] KIND * `-b, --branch TEXT`: Target branch * `--config-file TEXT`: [env var: INFRAHUBCTL_CONFIG; default: infrahubctl.toml] * `--help`: Show this message and exit. + +## `infrahubctl schema format` + +Format Infrahub schema files with a canonical key ordering. + +Reorders the keys within each node, generic, attribute, relationship and +dropdown choice into a consistent, opinionated order so schema files read +the same way and produce small diffs. + +Only your own nodes are formatted; nodes in Infrahub-reserved namespaces are +left untouched. Comments, quoting, and inline (flow) sequences are preserved. + +By default the change is purely key ordering. The opt-in flags additionally +change content: --strip-defaults drops redundant default values, +--sort-by-order-weight reorders attributes/relationships, and +--backfill-order-weight fills in a missing order_weight. + +Examples: + infrahubctl schema format schemas/ + infrahubctl schema format schemas/dcim.yml --diff + infrahubctl schema format schemas/ --check + infrahubctl schema format schemas/ --strip-defaults --sort-by-order-weight + +**Usage**: + +```console +$ infrahubctl schema format [OPTIONS] SCHEMAS... +``` + +**Arguments**: + +* `SCHEMAS...`: [required] + +**Options**: + +* `--check`: Do not write files; exit 1 if any file would be reformatted. +* `--diff`: Print a diff of the changes instead of writing files. +* `--strip-defaults`: Remove attribute/relationship/node keys whose value equals the schema default. +* `--sort-by-order-weight`: Sort attributes and relationships by order_weight (items without one keep their order and go last). +* `--backfill-order-weight`: Give attributes/relationships that lack an order_weight the value 1000. +* `--config-file TEXT`: [env var: INFRAHUBCTL_CONFIG; default: infrahubctl.toml] +* `--help`: Show this message and exit. diff --git a/infrahub_sdk/ctl/schema.py b/infrahub_sdk/ctl/schema.py index 21f9c7a98..004ada75f 100644 --- a/infrahub_sdk/ctl/schema.py +++ b/infrahub_sdk/ctl/schema.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import difflib import time from datetime import datetime, timezone from pathlib import Path @@ -19,6 +20,13 @@ from ..schema import NodeSchemaAPI, SchemaWarning from ..yaml import SchemaFile from .parameters import CONFIG_PARAM +from .schema_format import ( + DEFAULT_BACKFILL_ORDER_WEIGHT, + FormatError, + FormatOptions, + format_schema_text, + is_schema_document, +) from .utils import load_yamlfile_from_disk_and_exit if TYPE_CHECKING: @@ -414,3 +422,153 @@ async def schema_show( "Yes" if rel.optional else "No", ) console.print(rel_table) + + +def _print_schema_diff(location: Path, original: str, formatted: str) -> None: + diff = difflib.unified_diff( + original.splitlines(keepends=True), + formatted.splitlines(keepends=True), + fromfile=f"{location} (current)", + tofile=f"{location} (formatted)", + ) + for line in diff: + # markup=False keeps bracketed diff content (e.g. `[manufacturer, name]`) + # literal, so colour is applied via style= rather than inline markup. + if line.startswith("+") and not line.startswith("+++"): + console.print(line, end="", markup=False, highlight=False, style="green") + elif line.startswith("-") and not line.startswith("---"): + console.print(line, end="", markup=False, highlight=False, style="red") + else: + console.print(line, end="", markup=False, highlight=False) + + +def _format_one_schema_file( + location: Path, entries: list[SchemaFile], check: bool, diff: bool, options: FormatOptions +) -> str: + """Format a single schema file and report what happened. + + Args: + location: Path of the file on disk. + entries: SchemaFile entries parsed for this location (more than one means + a genuine multi-document file, which is not supported). + check: Report changes without writing. + diff: Print a diff instead of writing. + options: Opt-in transforms to apply. + + Returns: + One of ``"error"``, ``"skipped"``, ``"unchanged"`` or ``"changed"``. + + """ + if len(entries) > 1: + console.print(f"[yellow] Skipped {location}: multi-document files are not supported by format") + return "skipped" + + schema_file = entries[0] + if not schema_file.valid or schema_file.content is None: + console.print(f"[red] {location}: {schema_file.error_message or 'invalid file'}") + return "error" + + if not is_schema_document(schema_file.content): + return "skipped" + + original = location.read_text(encoding="utf-8") + try: + formatted = format_schema_text(original, options) + except FormatError as exc: + console.print(f"[red] {location}: {exc}") + return "error" + + if formatted == original: + return "unchanged" + + if diff: + _print_schema_diff(location=location, original=original, formatted=formatted) + elif check: + console.print(f"[yellow] Would reformat {location}") + else: + location.write_text(formatted, encoding="utf-8") + console.print(f"[green] Reformatted {location}") + return "changed" + + +@app.command(name="format") +@catch_exception(console=console) +def schema_format( + schemas: list[Path], + check: bool = typer.Option(False, "--check", help="Do not write files; exit 1 if any file would be reformatted."), + diff: bool = typer.Option(False, "--diff", help="Print a diff of the changes instead of writing files."), + strip_defaults: bool = typer.Option( + False, "--strip-defaults", help="Remove attribute/relationship/node keys whose value equals the schema default." + ), + sort_by_order_weight: bool = typer.Option( + False, + "--sort-by-order-weight", + help="Sort attributes and relationships by order_weight (items without one keep their order and go last).", + ), + backfill_order_weight: bool = typer.Option( + False, + "--backfill-order-weight", + help=f"Give attributes/relationships that lack an order_weight the value {DEFAULT_BACKFILL_ORDER_WEIGHT}.", + ), + _: str = CONFIG_PARAM, +) -> None: + """Format Infrahub schema files with a canonical key ordering. + + Reorders the keys within each node, generic, attribute, relationship and + dropdown choice into a consistent, opinionated order so schema files read + the same way and produce small diffs. + + Only your own nodes are formatted; nodes in Infrahub-reserved namespaces are + left untouched. Comments, quoting, and inline (flow) sequences are preserved. + + By default the change is purely key ordering. The opt-in flags additionally + change content: --strip-defaults drops redundant default values, + --sort-by-order-weight reorders attributes/relationships, and + --backfill-order-weight fills in a missing order_weight. + + \b + Examples: + infrahubctl schema format schemas/ + infrahubctl schema format schemas/dcim.yml --diff + infrahubctl schema format schemas/ --check + infrahubctl schema format schemas/ --strip-defaults --sort-by-order-weight + """ + options = FormatOptions( + strip_defaults=strip_defaults, + sort_by_order_weight=sort_by_order_weight, + backfill_order_weight=backfill_order_weight, + ) + schema_files = SchemaFile.load_from_disk(paths=schemas) + + # A genuine multi-document file yields several SchemaFile entries for the + # same location. The per-file ``multiple_documents`` flag is unreliable + # (it is set from a naive `---` substring count that also matches `---` + # inside comments), so group by location and count real documents instead. + entries_by_location: dict[Path, list[SchemaFile]] = {} + for schema_file in schema_files: + entries_by_location.setdefault(schema_file.location, []).append(schema_file) + + reformatted = 0 + unchanged = 0 + would_change = 0 + has_error = False + + for location, entries in entries_by_location.items(): + status = _format_one_schema_file(location=location, entries=entries, check=check, diff=diff, options=options) + if status == "error": + has_error = True + elif status == "unchanged": + unchanged += 1 + elif status == "changed": + if check or diff: + would_change += 1 + else: + reformatted += 1 + + if check or diff: + console.print(f"\n[bold]{would_change} file(s) would be reformatted, {unchanged} unchanged.") + else: + console.print(f"\n[bold]{reformatted} file(s) reformatted, {unchanged} unchanged.") + + if has_error or (check and would_change): + raise typer.Exit(1) diff --git a/infrahub_sdk/ctl/schema_drift.py b/infrahub_sdk/ctl/schema_drift.py new file mode 100644 index 000000000..b57457a83 --- /dev/null +++ b/infrahub_sdk/ctl/schema_drift.py @@ -0,0 +1,105 @@ +"""Detect drift between the Infrahub JSON schema and the formatter's baseline. + +The canonical key ordering in :mod:`infrahub_sdk.ctl.schema_format` is written +against a known set of schema properties. When Infrahub adds, removes, or +renames a property in the published JSON schema +(https://schema.infrahub.app/infrahub/schema/latest.json), that ordering may +need updating so the new key lands in a sensible slot rather than being +preserved as an unrecognised key. + +This module compares the live schema against a committed baseline +(``schema_properties.json``) and reports the difference. It backs the +``schema-drift-check`` invoke task (a warn-only CI step) and the +``schema-drift-update`` task that refreshes the baseline. It never raises on +drift — reporting is the caller's job. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import httpx + +from .schema_format import SCHEMA_URL + +# The JSON-schema ``$defs`` whose property sets the formatter orders. Each maps +# to a canonical key list in ``schema_format`` (nodes/generics, attributes, +# relationships, dropdown choices, and node extensions). +TRACKED_DEFINITIONS = [ + "NodeSchema", + "GenericSchema", + "AttributeSchema", + "RelationshipSchema", + "DropdownChoice", + "NodeExtensionSchema", +] + +BASELINE_PATH = Path(__file__).parent / "schema_properties.json" + + +def extract_properties(schema: dict[str, Any]) -> dict[str, list[str]]: + """Extract the sorted property names of each tracked definition. + + Args: + schema: The parsed JSON schema document. + + Returns: + A mapping of definition name to its sorted list of property names. + + """ + definitions = schema.get("$defs") or schema.get("definitions") or {} + return {name: sorted(definitions.get(name, {}).get("properties", {})) for name in TRACKED_DEFINITIONS} + + +def fetch_live_properties(url: str = SCHEMA_URL, timeout: float = 30.0) -> dict[str, list[str]]: + """Fetch the live JSON schema and return its tracked property sets. + + Args: + url: The schema URL to fetch. + timeout: Request timeout in seconds. + + Returns: + A mapping of definition name to its sorted list of property names. + + Raises: + httpx.HTTPError: If the schema cannot be fetched. + + """ + response = httpx.get(url, timeout=timeout, follow_redirects=True) + response.raise_for_status() + return extract_properties(response.json()) + + +def load_baseline(path: Path = BASELINE_PATH) -> dict[str, list[str]]: + """Load the committed baseline property sets.""" + return json.loads(path.read_text(encoding="utf-8")) + + +def write_baseline(properties: dict[str, list[str]], path: Path = BASELINE_PATH) -> None: + """Write ``properties`` to the baseline file as sorted, indented JSON.""" + path.write_text(json.dumps(properties, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def compute_drift(live: dict[str, list[str]], baseline: dict[str, list[str]]) -> dict[str, dict[str, list[str]]]: + """Compare live and baseline property sets. + + Args: + live: Property sets from the live schema. + baseline: Property sets from the committed baseline. + + Returns: + A mapping of definition name to ``{"added": [...], "removed": [...]}``, + containing only the definitions that changed. + + """ + drift: dict[str, dict[str, list[str]]] = {} + for name in TRACKED_DEFINITIONS: + live_set = set(live.get(name, [])) + baseline_set = set(baseline.get(name, [])) + added = sorted(live_set - baseline_set) + removed = sorted(baseline_set - live_set) + if added or removed: + drift[name] = {"added": added, "removed": removed} + return drift diff --git a/infrahub_sdk/ctl/schema_format.py b/infrahub_sdk/ctl/schema_format.py new file mode 100644 index 000000000..530416b18 --- /dev/null +++ b/infrahub_sdk/ctl/schema_format.py @@ -0,0 +1,431 @@ +"""Opinionated formatter for Infrahub schema YAML files. + +The formatter's core responsibility is the *ordering of keys* (lines) within +each node, generic, attribute, relationship and dropdown choice, so that +hand-authored schema files read consistently and produce small diffs. + +By default the transformation is purely cosmetic and semantics-preserving: +only line order changes, and :func:`format_schema_text` re-parses its own +output and raises if the reloaded data differs from the input. + +Three opt-in transforms (see :class:`FormatOptions`) go further and *do* change +what is written — each is off by default and neutralised in the safety check so +only its intended effect is allowed: + +- ``strip_defaults`` — drop keys whose value equals the schema default (context + aware: ``optional: true`` is redundant on a relationship but meaningful on an + attribute). +- ``sort_by_order_weight`` — sort attributes and relationships ascending by + ``order_weight``; items without one keep their authored order and go last. +- ``backfill_order_weight`` — give attributes/relationships that lack an + ``order_weight`` a single constant value. + +Formatting is done with ``ruamel.yaml`` in round-trip mode, so comments (the +``# yaml-language-server`` header, standalone notes, and inline comments), +quoting style, and flow-style sequences (e.g. ``[manufacturer, name__value]``) +are preserved. Standalone comments sitting *between* attributes/relationships +may not follow their item when ``sort_by_order_weight`` reorders the list; +inline comments on a value always travel with it. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from io import StringIO +from typing import Any + +import yaml +from ruamel.yaml import YAML + +# Mirrors ``infrahub.core.constants.RESTRICTED_NAMESPACES``. Kept as a local +# copy because the SDK does not depend on the Infrahub backend. This list is +# stable; if it drifts, a node in a newly restricted namespace would simply be +# formatted like a user node (a harmless outcome for a line-ordering tool). +RESTRICTED_NAMESPACES: list[str] = [ + "Account", + "Branch", + "Builtin", + "Core", + "Deprecated", + "Diff", + "Infrahub", + "Internal", + "Lineage", + "Schema", + "Profile", + "Template", +] + +SCHEMA_URL = "https://schema.infrahub.app/infrahub/schema/latest.json" +SCHEMA_HEADER = f"---\n# yaml-language-server: $schema={SCHEMA_URL}\n" + +# order_weight has no numeric default in the schema (the UI falls back to +# declaration order), so backfill writes this constant. +DEFAULT_BACKFILL_ORDER_WEIGHT = 1000 + +# Matches a real yaml-language-server directive: a comment line whose first +# non-whitespace content is ``# yaml-language-server:``. Deliberately does not +# match the substring appearing in a scalar value or an unrelated comment. +_LANGUAGE_SERVER_HEADER_RE = re.compile(r"^[ \t]*#[ \t]*yaml-language-server[ \t]*:", re.MULTILINE) + +# Canonical key orders. Each pair is (leading keys, trailing keys); any key not +# listed is preserved in its original position between the two groups so the +# formatter never drops data. +FILE_ORDER = ["version", "generics", "nodes", "extensions"] + +NODE_ORDER = [ + "name", + "namespace", + "description", + "label", + "icon", + "documentation", + "include_in_menu", + "menu_placement", + "inherit_from", + "parent", + "children", + "hierarchical", + "default_filter", + "human_friendly_id", + "order_by", + "display_label", + "display_labels", + "uniqueness_constraints", + "generate_profile", + "generate_template", + "used_by", + "restricted_namespaces", + "branch", + "state", +] +NODE_LAST = ["attributes", "relationships"] + +ATTRIBUTE_ORDER = [ + "name", + "kind", + "label", + "unique", + "read_only", + "computed_attribute", + "default_value", + "enum", + "choices", + "regex", + "min_length", + "max_length", + "parameters", + "optional", + "description", + "allow_override", + "branch", + "deprecation", + "state", +] +ATTRIBUTE_LAST = ["order_weight"] + +RELATIONSHIP_ORDER = [ + "name", + "peer", + "label", + "kind", + "cardinality", + "optional", + "identifier", + "direction", + "on_delete", + "hierarchical", + "min_count", + "max_count", + "common_parent", + "common_relatives", + "read_only", + "allow_override", + "branch", + "deprecation", + "state", + "description", +] +RELATIONSHIP_LAST = ["order_weight"] + +CHOICE_ORDER = ["name", "label", "description", "color"] + +EXTENSION_NODE_ORDER = ["kind", "inherit_from"] +EXTENSION_NODE_LAST = ["attributes", "relationships"] + +# Strippable defaults, grounded in the published JSON schema's ``default`` +# values. Consequential or internal fields (``branch``, ``state``, +# ``inherited``, ``display``) are intentionally excluded: stripping an explicit +# value there would couple the schema to whatever the default happens to be at +# load time. +ENTITY_DEFAULTS: dict[str, Any] = { + "generate_profile": True, + "generate_template": False, + "hierarchical": False, +} +ATTRIBUTE_DEFAULTS: dict[str, Any] = { + "read_only": False, + "unique": False, + "optional": False, + "allow_override": "any", +} +RELATIONSHIP_DEFAULTS: dict[str, Any] = { + "kind": "Generic", + "cardinality": "many", + "optional": True, + "direction": "bidirectional", + "read_only": False, + "allow_override": "any", + "min_count": 0, + "max_count": 0, +} + + +@dataclass(frozen=True) +class FormatOptions: + """Opt-in transforms that change file content beyond key ordering.""" + + strip_defaults: bool = False + sort_by_order_weight: bool = False + backfill_order_weight: bool = False + + +class FormatError(Exception): + """Raised when formatting would change the meaning of a schema file.""" + + +def _build_yaml() -> YAML: + """Return a round-trip YAML handler configured to match the schema-library style.""" + yaml_handler = YAML() + yaml_handler.preserve_quotes = True + # Schema files begin with a `---` document-start marker; keep it. + yaml_handler.explicit_start = True + # Match the schema-library layout: block sequences indented under their key + # (`attributes:\n - name: ...`). + yaml_handler.indent(mapping=2, sequence=4, offset=2) + # A very wide value keeps long scalars (descriptions, Jinja2 templates) on + # their original line instead of being re-wrapped. + yaml_handler.width = 4096 + return yaml_handler + + +def reorder_mapping(mapping: Any, leading: list[str], trailing: list[str]) -> None: + """Reorder a mapping's keys in place into canonical order. + + Keys in ``leading`` come first (in that order), keys in ``trailing`` come + last (in that order), and any remaining keys keep their original relative + order in between. Reordering is done in place with ``move_to_end`` so the + comments ruamel attaches to each key travel with it. + + Args: + mapping: The (round-trip) mapping to reorder. + leading: Keys to place first, in order. + trailing: Keys to force to the end, in order. + + """ + # Round-trip maps (and OrderedDict) support move_to_end; anything else + # (a scalar, a plain list) is left as-is. + if not hasattr(mapping, "move_to_end"): + return + + known = set(leading) | set(trailing) + ordered_keys = [key for key in leading if key in mapping] + ordered_keys += [key for key in mapping if key not in known] + ordered_keys += [key for key in trailing if key in mapping] + + for key in ordered_keys: + mapping.move_to_end(key) + + +def _strip_default_keys(mapping: Any, defaults: dict[str, Any]) -> None: + """Remove keys whose value equals the schema default.""" + if not isinstance(mapping, dict): + return + for key, default in defaults.items(): + if key in mapping and mapping[key] == default: + del mapping[key] + + +def _order_weight_sort_key(item: Any) -> float: + weight = item.get("order_weight") if isinstance(item, dict) else None + # Missing/non-numeric weights sort last; a stable sort keeps their order. + return weight if isinstance(weight, int) and not isinstance(weight, bool) else float("inf") + + +def _format_item(item: Any, defaults: dict[str, Any], leading: list[str], options: FormatOptions) -> None: + if not isinstance(item, dict): + return + if options.backfill_order_weight and "order_weight" not in item: + item["order_weight"] = DEFAULT_BACKFILL_ORDER_WEIGHT + if options.strip_defaults: + _strip_default_keys(item, defaults) + reorder_mapping(item, leading, ["order_weight"]) + if defaults is ATTRIBUTE_DEFAULTS: + choices = item.get("choices") + if isinstance(choices, list): + for choice in choices: + reorder_mapping(choice, CHOICE_ORDER, []) + + +def _format_item_list(items: Any, defaults: dict[str, Any], leading: list[str], options: FormatOptions) -> None: + if not isinstance(items, list): + return + for item in items: + _format_item(item, defaults, leading, options) + if options.sort_by_order_weight: + items.sort(key=_order_weight_sort_key) + + +def _format_entity(entity: Any, leading: list[str], trailing: list[str], options: FormatOptions) -> None: + """Reorder an entity's own keys, then transform its attributes and relationships.""" + if options.strip_defaults: + _strip_default_keys(entity, ENTITY_DEFAULTS) + reorder_mapping(entity, leading, trailing) + _format_item_list(entity.get("attributes"), ATTRIBUTE_DEFAULTS, ATTRIBUTE_ORDER, options) + _format_item_list(entity.get("relationships"), RELATIONSHIP_DEFAULTS, RELATIONSHIP_ORDER, options) + + +def _is_restricted(entity: Any) -> bool: + return isinstance(entity, dict) and entity.get("namespace") in RESTRICTED_NAMESPACES + + +def format_document(data: Any, options: FormatOptions | None = None) -> None: + """Reorder (and optionally transform) a parsed schema document in place. + + Nodes and generics in a restricted namespace are left untouched. Extension + entries are always formatted, since the extension block itself is authored + by the user regardless of which node it extends. + + Args: + data: The parsed (round-trip) schema document. + options: Opt-in transforms; defaults to key-ordering only. + + """ + options = options or FormatOptions() + reorder_mapping(data, FILE_ORDER, []) + + for section in ("generics", "nodes"): + entities = data.get(section) + if not isinstance(entities, list): + continue + for entity in entities: + if isinstance(entity, dict) and not _is_restricted(entity): + _format_entity(entity, NODE_ORDER, NODE_LAST, options) + + extensions = data.get("extensions") + if isinstance(extensions, dict) and isinstance(extensions.get("nodes"), list): + for entity in extensions["nodes"]: + if isinstance(entity, dict): + _format_entity(entity, EXTENSION_NODE_ORDER, EXTENSION_NODE_LAST, options) + + +def _ensure_schema_header(text: str) -> str: + """Add the canonical ``# yaml-language-server`` header if the file lacks one. + + Only an actual header *directive line* counts as present — a bare + ``yaml-language-server`` substring elsewhere (in a scalar value or an + unrelated comment) must not suppress the header. + """ + if _LANGUAGE_SERVER_HEADER_RE.search(text): + return text + if text.startswith("---\n"): + return SCHEMA_HEADER + text[len("---\n") :] + return SCHEMA_HEADER + text + + +def is_schema_document(content: Any) -> bool: + """Return True if ``content`` looks like an Infrahub schema file.""" + return ( + isinstance(content, dict) + and "version" in content + and any(key in content for key in ("nodes", "generics", "extensions")) + ) + + +def _normalize_item(item: Any, defaults: dict[str, Any], options: FormatOptions) -> Any: + if not isinstance(item, dict): + return item + normalized = dict(item) + if options.strip_defaults: + for key, default in defaults.items(): + normalized.setdefault(key, default) + if options.backfill_order_weight: + normalized.setdefault("order_weight", DEFAULT_BACKFILL_ORDER_WEIGHT) + return normalized + + +def _normalize_entity(entity: Any, options: FormatOptions) -> Any: + if not isinstance(entity, dict): + return entity + normalized = dict(entity) + if options.strip_defaults: + for key, default in ENTITY_DEFAULTS.items(): + normalized.setdefault(key, default) + for key, defaults in (("attributes", ATTRIBUTE_DEFAULTS), ("relationships", RELATIONSHIP_DEFAULTS)): + items = normalized.get(key) + if isinstance(items, list): + items = [_normalize_item(item, defaults, options) for item in items] + if options.sort_by_order_weight: + items = sorted(items, key=lambda it: it.get("name", "") if isinstance(it, dict) else "") + normalized[key] = items + return normalized + + +def _normalize_for_guard(data: Any, options: FormatOptions) -> Any: + """Collapse exactly the intended transforms so the guard permits them. + + The same normalisation is applied to the input and the formatted output, so + an intended change (a stripped default, a reordered list, a backfilled + weight) is neutralised on both sides while any *unintended* corruption still + causes inequality. With no options set this is effectively an identity. + """ + if not isinstance(data, dict): + return data + normalized = dict(data) + for section in ("generics", "nodes"): + entities = normalized.get(section) + if isinstance(entities, list): + normalized[section] = [_normalize_entity(entity, options) for entity in entities] + extensions = normalized.get("extensions") + if isinstance(extensions, dict) and isinstance(extensions.get("nodes"), list): + extensions = dict(extensions) + extensions["nodes"] = [_normalize_entity(entity, options) for entity in extensions["nodes"]] + normalized["extensions"] = extensions + return normalized + + +def format_schema_text(raw_text: str, options: FormatOptions | None = None) -> str: + """Format the text of a schema file into canonical YAML text. + + Args: + raw_text: The original file contents. + options: Opt-in transforms; defaults to key-ordering only. + + Returns: + The formatted YAML text, with comments and quoting preserved. + + Raises: + FormatError: If formatting would change the file's meaning beyond the + transforms requested via ``options``. + + """ + options = options or FormatOptions() + yaml_handler = _build_yaml() + data = yaml_handler.load(raw_text) + + if not is_schema_document(data): + return raw_text + + format_document(data, options) + + buffer = StringIO() + yaml_handler.dump(data, buffer) + text = _ensure_schema_header(buffer.getvalue()) + + original = _normalize_for_guard(yaml.safe_load(raw_text), options) + formatted = _normalize_for_guard(yaml.safe_load(text), options) + if original != formatted: + raise FormatError("Formatting would change the schema content; aborting to avoid data loss.") + + return text diff --git a/infrahub_sdk/ctl/schema_properties.json b/infrahub_sdk/ctl/schema_properties.json new file mode 100644 index 000000000..ae2876d82 --- /dev/null +++ b/infrahub_sdk/ctl/schema_properties.json @@ -0,0 +1,120 @@ +{ + "AttributeSchema": [ + "allow_override", + "branch", + "choices", + "computed_attribute", + "default_value", + "deprecation", + "description", + "display", + "enum", + "id", + "inherited", + "kind", + "label", + "max_length", + "min_length", + "name", + "optional", + "order_weight", + "parameters", + "read_only", + "regex", + "state", + "unique" + ], + "DropdownChoice": [ + "color", + "description", + "id", + "label", + "name", + "state" + ], + "GenericSchema": [ + "attributes", + "branch", + "default_filter", + "description", + "display_label", + "display_labels", + "documentation", + "generate_profile", + "hierarchical", + "human_friendly_id", + "icon", + "id", + "include_in_menu", + "label", + "menu_placement", + "name", + "namespace", + "order_by", + "relationships", + "restricted_namespaces", + "state", + "uniqueness_constraints", + "used_by" + ], + "NodeExtensionSchema": [ + "attributes", + "id", + "kind", + "relationships", + "state" + ], + "NodeSchema": [ + "attributes", + "branch", + "children", + "default_filter", + "description", + "display_label", + "display_labels", + "documentation", + "generate_profile", + "generate_template", + "hierarchy", + "human_friendly_id", + "icon", + "id", + "include_in_menu", + "inherit_from", + "label", + "menu_placement", + "name", + "namespace", + "order_by", + "parent", + "relationships", + "state", + "uniqueness_constraints" + ], + "RelationshipSchema": [ + "allow_override", + "branch", + "cardinality", + "common_parent", + "common_relatives", + "deprecation", + "description", + "direction", + "display", + "hierarchical", + "id", + "identifier", + "inherited", + "kind", + "label", + "max_count", + "min_count", + "name", + "on_delete", + "optional", + "order_weight", + "peer", + "read_only", + "state" + ] +} diff --git a/pyproject.toml b/pyproject.toml index 5a8bbf31b..838a378f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ ctl = [ "numpy>=1.26.2; python_version>='3.12'", "pyarrow>=14", "pyyaml>=6", + "ruamel.yaml>=0.18", "rich>=12,<14", "typer>=0.15.0", "click>=8.3,<9", @@ -61,6 +62,7 @@ all = [ "pyarrow>=14", "pytest", "pyyaml>=6", + "ruamel.yaml>=0.18", "rich>=12,<14", "typer>=0.15.0", "click>=8.3,<9", diff --git a/tasks.py b/tasks.py index 69bf79860..b29b1d695 100644 --- a/tasks.py +++ b/tasks.py @@ -427,3 +427,41 @@ def generate_repository_jsonschema(context: Context) -> None: repository_jsonschema.parent.mkdir(parents=True, exist_ok=True) repository_jsonschema.write_text(schema) print(f"Wrote to {repository_jsonschema}") + + +@task(name="schema-drift-check") +def schema_drift_check(context: Context) -> None: # noqa: ARG001 + """Warn (without failing) if the live Infrahub JSON schema drifted from the committed baseline. + + Emits GitHub Actions ``::warning::`` annotations for any added or removed + schema property so the formatter's canonical key ordering in + ``infrahub_sdk/ctl/schema_format.py`` can be updated. Always exits 0. + """ + from infrahub_sdk.ctl.schema_drift import compute_drift, fetch_live_properties, load_baseline + + try: + live = fetch_live_properties() + except Exception as exc: + print(f"::warning title=Schema drift check::Could not fetch the Infrahub schema: {exc}") + return + + drift = compute_drift(live=live, baseline=load_baseline()) + if not drift: + print("Infrahub schema is in sync with the committed baseline; no drift detected.") + return + + hint = "update infrahub_sdk/ctl/schema_format.py if needed, then run 'invoke schema-drift-update'" + for definition, change in drift.items(): + for prop in change["added"]: + print(f"::warning title=Schema drift::New schema property {definition}.{prop} — {hint}") + for prop in change["removed"]: + print(f"::warning title=Schema drift::Removed schema property {definition}.{prop} — {hint}") + + +@task(name="schema-drift-update") +def schema_drift_update(context: Context) -> None: # noqa: ARG001 + """Refresh infrahub_sdk/ctl/schema_properties.json from the live Infrahub JSON schema.""" + from infrahub_sdk.ctl.schema_drift import BASELINE_PATH, fetch_live_properties, write_baseline + + write_baseline(fetch_live_properties()) + print(f"Updated {BASELINE_PATH}") diff --git a/tests/unit/ctl/test_schema_drift.py b/tests/unit/ctl/test_schema_drift.py new file mode 100644 index 000000000..d81f1f0f0 --- /dev/null +++ b/tests/unit/ctl/test_schema_drift.py @@ -0,0 +1,52 @@ +"""Unit tests for the schema drift-detection logic (offline).""" + +from __future__ import annotations + +import json + +from infrahub_sdk.ctl.schema_drift import ( + BASELINE_PATH, + TRACKED_DEFINITIONS, + compute_drift, + extract_properties, + load_baseline, +) + + +def test_extract_properties_reads_defs() -> None: + schema = { + "$defs": { + "NodeSchema": {"properties": {"name": {}, "namespace": {}}}, + "AttributeSchema": {"properties": {"kind": {}, "name": {}}}, + } + } + result = extract_properties(schema) + + # Every tracked definition is present; values are sorted; unknown defs empty. + assert set(result) == set(TRACKED_DEFINITIONS) + assert result["NodeSchema"] == ["name", "namespace"] + assert result["AttributeSchema"] == ["kind", "name"] + assert result["RelationshipSchema"] == [] + + +def test_compute_drift_detects_added_and_removed() -> None: + baseline = {"NodeSchema": ["name", "namespace", "label"]} + live = {"NodeSchema": ["name", "namespace", "new_field"]} + + drift = compute_drift(live=live, baseline=baseline) + + assert drift == {"NodeSchema": {"added": ["new_field"], "removed": ["label"]}} + + +def test_compute_drift_empty_when_in_sync() -> None: + props = {"NodeSchema": ["name", "namespace"]} + assert compute_drift(live=props, baseline=props) == {} + + +def test_committed_baseline_is_valid_and_complete() -> None: + baseline = load_baseline() + # The shipped baseline covers exactly the tracked definitions and is JSON. + assert set(baseline) == set(TRACKED_DEFINITIONS) + assert all(isinstance(props, list) for props in baseline.values()) + # Round-trips through json (guards against a hand-edit breaking the file). + assert json.loads(BASELINE_PATH.read_text(encoding="utf-8")) == baseline diff --git a/tests/unit/ctl/test_schema_format.py b/tests/unit/ctl/test_schema_format.py new file mode 100644 index 000000000..25f2b1d56 --- /dev/null +++ b/tests/unit/ctl/test_schema_format.py @@ -0,0 +1,403 @@ +"""Unit tests for the pure schema-formatting logic in ``schema_format``.""" + +from __future__ import annotations + +from collections import OrderedDict + +import pytest +import yaml + +from infrahub_sdk.ctl.schema_format import ( + FormatError, + FormatOptions, + format_schema_text, + is_schema_document, + reorder_mapping, +) + +NODE_DOC = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" + +nodes: + - relationships: + - peer: BuiltinTag + name: tags + attributes: + - order_weight: 1500 + optional: true + name: status + kind: Dropdown + choices: + - color: "#fff" + name: active + label: Active + namespace: Dcim + name: Device + label: Device + description: A device. +""" + + +def _keys_of(text: str, path: list) -> list[str]: + """Load formatted YAML and return the key order of the mapping at ``path``.""" + data = yaml.safe_load(text) + for step in path: + data = data[step] + return list(data.keys()) + + +def test_reorder_mapping_leading_trailing_and_unknown() -> None: + data = OrderedDict([("order_weight", 1000), ("extra", "x"), ("kind", "Text"), ("name", "field")]) + reorder_mapping(data, leading=["name", "kind"], trailing=["order_weight"]) + + # name/kind first, order_weight last, unknown key preserved in the middle. + assert list(data.keys()) == ["name", "kind", "extra", "order_weight"] + + +def test_node_key_order_is_canonical() -> None: + text = format_schema_text(NODE_DOC) + + # Top-level sections: version before nodes. + assert _keys_of(text, [])[:2] == ["version", "nodes"] + # name/namespace first; attributes then relationships always last. + assert _keys_of(text, ["nodes", 0]) == [ + "name", + "namespace", + "description", + "label", + "attributes", + "relationships", + ] + + +def test_attribute_relationship_and_choice_inner_order() -> None: + text = format_schema_text(NODE_DOC) + + attr_keys = _keys_of(text, ["nodes", 0, "attributes", 0]) + assert attr_keys == ["name", "kind", "choices", "optional", "order_weight"] + assert attr_keys[-1] == "order_weight" + + assert _keys_of(text, ["nodes", 0, "attributes", 0, "choices", 0]) == ["name", "label", "color"] + assert _keys_of(text, ["nodes", 0, "relationships", 0]) == ["name", "peer"] + + +def test_restricted_namespace_nodes_are_untouched() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Core + name: Something + attributes: + - order_weight: 1 + kind: Text + name: x + - namespace: Dcim + name: Device + attributes: + - order_weight: 1 + kind: Text + name: x +""" + text = format_schema_text(doc) + + # Core node keeps its authored (scrambled) attribute key order. + assert _keys_of(text, ["nodes", 0, "attributes", 0]) == ["order_weight", "kind", "name"] + # Dcim (user) node is reordered. + assert _keys_of(text, ["nodes", 1, "attributes", 0]) == ["name", "kind", "order_weight"] + + +def test_extensions_are_formatted() -> None: + doc = """\ +--- +version: "1.0" +extensions: + nodes: + - relationships: + - peer: LocationSite + name: sites + kind: OrganizationProvider +""" + text = format_schema_text(doc) + assert _keys_of(text, ["extensions", "nodes", 0]) == ["kind", "relationships"] + assert _keys_of(text, ["extensions", "nodes", 0, "relationships", 0]) == ["name", "peer"] + + +def test_unknown_keys_are_preserved_not_dropped() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + some_future_key: value +""" + text = format_schema_text(doc) + assert _keys_of(text, ["nodes", 0]) == ["name", "namespace", "some_future_key"] + assert yaml.safe_load(text)["nodes"][0]["some_future_key"] == "value" + + +def test_comments_are_preserved() -> None: + doc = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" + +nodes: + # a banner comment before the node + - namespace: Dcim + name: Device + attributes: + - name: status + kind: Dropdown + choices: + - name: active + color: "#7fbf7f" # a trailing inline comment +""" + text = format_schema_text(doc) + assert "# a banner comment before the node" in text + assert "# a trailing inline comment" in text + assert "yaml-language-server" in text + + +def test_flow_style_sequences_are_preserved() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + uniqueness_constraints: + - [manufacturer, name__value] +""" + text = format_schema_text(doc) + # The inline (flow) sequence is not expanded to block style. + assert "[manufacturer, name__value]" in text + + +def test_quotes_are_preserved() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + description: "A quoted description" +""" + text = format_schema_text(doc) + assert 'description: "A quoted description"' in text + assert 'version: "1.0"' in text + + +def test_multiline_string_round_trips() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + attributes: + - name: computed + kind: Text + read_only: true + computed_attribute: + kind: Jinja2 + jinja2_template: >- + {{ a__value }}-{{ b__value }} +""" + text = format_schema_text(doc) + assert yaml.safe_load(text) == yaml.safe_load(doc) + + +def test_header_is_added_when_missing() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim +""" + text = format_schema_text(doc) + assert text.startswith("---\n# yaml-language-server:") + + +def test_format_is_idempotent_and_semantics_preserved() -> None: + once = format_schema_text(NODE_DOC) + assert format_schema_text(once) == once + assert yaml.safe_load(once) == yaml.safe_load(NODE_DOC) + + +def test_non_schema_document_is_returned_unchanged() -> None: + doc = "apiVersion: infrahub.app/v1\nkind: Menu\nspec:\n data: []\n" + assert format_schema_text(doc) == doc + + +def test_format_error_raised_on_semantic_drift(monkeypatch: pytest.MonkeyPatch) -> None: + # Simulate a formatting step that silently drops data; the guard must catch it. + def _wipe(data: dict, options: object = None) -> None: + data.clear() + + monkeypatch.setattr("infrahub_sdk.ctl.schema_format.format_document", _wipe) + + with pytest.raises(FormatError): + format_schema_text(NODE_DOC) + + +def test_malformed_non_list_attributes_is_left_untouched() -> None: + # A parseable schema whose `attributes` is not a list must not crash the + # formatter; that section is simply left as-is. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: 5 + relationships: not-a-list +""" + text = format_schema_text(doc) + assert yaml.safe_load(text) == yaml.safe_load(doc) + + +def test_header_not_added_when_substring_appears_in_scalar() -> None: + # `yaml-language-server` appearing in a value (not as a real header line) + # must not suppress the header being added. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + description: see the yaml-language-server extension docs +""" + text = format_schema_text(doc) + assert text.startswith("---\n# yaml-language-server: $schema=") + # The real directive appears exactly once (added, not duplicated later). + assert text.count("# yaml-language-server:") == 1 + + +def test_existing_header_is_not_duplicated() -> None: + doc = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" +nodes: + - namespace: Dcim + name: Device +""" + text = format_schema_text(doc) + assert text.count("# yaml-language-server:") == 1 + + +STRIP_DOC = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: a + kind: Text + optional: false + - name: b + kind: Text + optional: true + relationships: + - name: r1 + peer: DcimX + optional: true + cardinality: many + kind: Generic + - name: r2 + peer: DcimY + optional: false + cardinality: one + kind: Attribute +""" + + +def test_strip_defaults_removes_only_default_values() -> None: + node = yaml.safe_load(format_schema_text(STRIP_DOC, FormatOptions(strip_defaults=True)))["nodes"][0] + attrs = {a["name"]: a for a in node["attributes"]} + rels = {r["name"]: r for r in node["relationships"]} + + # Attribute default optional:false stripped; non-default optional:true kept. + assert "optional" not in attrs["a"] + assert attrs["b"]["optional"] is True + + # Relationship defaults (optional:true, cardinality:many, kind:Generic) stripped. + assert set(rels["r1"].keys()) == {"name", "peer"} + # Non-default relationship values are kept. + assert rels["r2"]["optional"] is False + assert rels["r2"]["cardinality"] == "one" + assert rels["r2"]["kind"] == "Attribute" + + +SORT_DOC = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: c + kind: Text + order_weight: 3000 + - name: a + kind: Text + order_weight: 1000 + - name: b + kind: Text + - name: d + kind: Text + order_weight: 2000 +""" + + +def test_sort_by_order_weight_ascending_missing_last() -> None: + node = yaml.safe_load(format_schema_text(SORT_DOC, FormatOptions(sort_by_order_weight=True)))["nodes"][0] + names = [a["name"] for a in node["attributes"]] + # Weighted ascending (a=1000, d=2000, c=3000), then the weightless one last. + assert names == ["a", "d", "c", "b"] + + +def test_backfill_order_weight_only_fills_missing() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: a + kind: Text + - name: b + kind: Text + order_weight: 5 +""" + node = yaml.safe_load(format_schema_text(doc, FormatOptions(backfill_order_weight=True)))["nodes"][0] + weights = {a["name"]: a["order_weight"] for a in node["attributes"]} + assert weights == {"a": 1000, "b": 5} + + +def test_flags_are_idempotent_and_off_by_default() -> None: + # Off by default: no content change beyond ordering (STRIP_DOC has a + # strippable default that must survive when the flag is not set). + default_out = yaml.safe_load(format_schema_text(STRIP_DOC)) + assert default_out["nodes"][0]["attributes"][0].get("optional") is False + + opts = FormatOptions(strip_defaults=True, sort_by_order_weight=True, backfill_order_weight=True) + once = format_schema_text(STRIP_DOC, opts) + assert format_schema_text(once, opts) == once + + +def test_is_schema_document() -> None: + assert is_schema_document({"version": "1.0", "nodes": []}) + assert is_schema_document({"version": "1.0", "generics": []}) + assert is_schema_document({"version": "1.0", "extensions": {}}) + assert not is_schema_document({"version": "1.0"}) + assert not is_schema_document({"nodes": []}) + assert not is_schema_document({"apiVersion": "infrahub.app/v1", "kind": "Menu"}) + assert not is_schema_document("not a dict") diff --git a/tests/unit/ctl/test_schema_format_app.py b/tests/unit/ctl/test_schema_format_app.py new file mode 100644 index 000000000..a4e5ad5f5 --- /dev/null +++ b/tests/unit/ctl/test_schema_format_app.py @@ -0,0 +1,193 @@ +"""CLI tests for ``infrahubctl schema format``.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from typer.testing import CliRunner + +from infrahub_sdk.ctl.schema import app +from tests.helpers.cli import remove_ansi_color + +runner = CliRunner() + +# Widen the Rich console so long tmp_path locations are not wrapped across +# lines, which would break substring assertions on the output. +WIDE = {"COLUMNS": "300"} + + +UNFORMATTED = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + # a design note + label: Device + attributes: + - order_weight: 1000 + kind: Text + name: name + unique: true +""" + + +def _write(path: Path, content: str) -> Path: + path.write_text(content, encoding="utf-8") + return path + + +def test_format_writes_in_place(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(schema)]) + + assert result.exit_code == 0 + output = remove_ansi_color(result.stdout) + assert f"Reformatted {schema}" in output + assert "1 file(s) reformatted" in output + + formatted = schema.read_text(encoding="utf-8") + # Header re-added, keys reordered (name before kind, order_weight last). + assert formatted.startswith("---\n# yaml-language-server:") + name_idx = formatted.index("name: name") + kind_idx = formatted.index("kind: Text") + weight_idx = formatted.index("order_weight: 1000") + assert name_idx < kind_idx < weight_idx + + +def test_format_is_idempotent(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + once = schema.read_text(encoding="utf-8") + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + twice = schema.read_text(encoding="utf-8") + + assert once == twice + + +def test_format_check_reports_and_exits_nonzero(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + before = schema.read_text(encoding="utf-8") + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--check"]) + + assert result.exit_code == 1 + assert "Would reformat" in remove_ansi_color(result.stdout) + # --check never writes. + assert schema.read_text(encoding="utf-8") == before + + +def test_format_check_clean_file_exits_zero(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + runner.invoke(app, env=WIDE, args=["format", str(schema)]) # normalise first + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--check"]) + + assert result.exit_code == 0 + assert "0 file(s) would be reformatted" in remove_ansi_color(result.stdout) + + +def test_format_diff_prints_and_does_not_write(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + before = schema.read_text(encoding="utf-8") + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--diff"]) + + assert result.exit_code == 0 + output = remove_ansi_color(result.stdout) + assert "yaml-language-server" in output # the added header shows up in the diff + # Colour is applied via Rich styling, not literal markup tags. + assert "[green]" not in output + assert "[red]" not in output + assert schema.read_text(encoding="utf-8") == before + + +def test_format_preserves_comments(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(schema)]) + + assert result.exit_code == 0 + # The comment survives the reformat. + assert "# a design note" in schema.read_text(encoding="utf-8") + + +def test_format_skips_non_schema_yaml(tmp_path: Path) -> None: + menu = _write(tmp_path / "menu.yml", "apiVersion: infrahub.app/v1\nkind: Menu\nspec:\n data: []\n") + + result = runner.invoke(app, env=WIDE, args=["format", str(menu)]) + + assert result.exit_code == 0 + assert "0 file(s) reformatted, 0 unchanged" in remove_ansi_color(result.stdout) + + +def test_format_directory_recurses(tmp_path: Path) -> None: + _write(tmp_path / "a.yml", UNFORMATTED) + _write(tmp_path / "b.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(tmp_path)]) + + assert result.exit_code == 0 + assert "2 file(s) reformatted" in remove_ansi_color(result.stdout) + + +def test_format_opt_in_flags(tmp_path: Path) -> None: + content = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + relationships: + - name: b_rel + peer: DcimB + optional: true + cardinality: many + order_weight: 2000 + - name: a_rel + peer: DcimA + kind: Attribute + cardinality: one +""" + schema = _write(tmp_path / "dcim.yml", content) + + result = runner.invoke( + app, + env=WIDE, + args=["format", str(schema), "--strip-defaults", "--sort-by-order-weight", "--backfill-order-weight"], + ) + + assert result.exit_code == 0 + out = schema.read_text(encoding="utf-8") + rels = yaml.safe_load(out)["nodes"][0]["relationships"] + # backfill filled a_rel (was missing) with 1000, so it sorts before b_rel (2000). + assert [r["name"] for r in rels] == ["a_rel", "b_rel"] + # strip-defaults removed the redundant optional:true / cardinality:many on b_rel. + assert "optional" not in rels[1] + assert "cardinality" not in rels[1] + + +def test_format_leaves_restricted_namespace_untouched(tmp_path: Path) -> None: + content = """\ +--- +version: "1.0" +nodes: + - namespace: Core + name: Special + attributes: + - order_weight: 1000 + kind: Text + name: name +""" + schema = _write(tmp_path / "core.yml", content) + + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + formatted = schema.read_text(encoding="utf-8") + + # The Core node's attribute keys keep their original (scrambled) order. + weight_idx = formatted.index("order_weight: 1000") + name_idx = formatted.index("name: name") + assert weight_idx < name_idx diff --git a/uv.lock b/uv.lock index a0b1cd60e..95284517d 100644 --- a/uv.lock +++ b/uv.lock @@ -700,6 +700,7 @@ all = [ { name = "pytest" }, { name = "pyyaml" }, { name = "rich" }, + { name = "ruamel-yaml" }, { name = "typer" }, ] ctl = [ @@ -712,6 +713,7 @@ ctl = [ { name = "pyarrow" }, { name = "pyyaml" }, { name = "rich" }, + { name = "ruamel-yaml" }, { name = "typer" }, ] @@ -790,6 +792,8 @@ requires-dist = [ { name = "pyyaml", marker = "extra == 'ctl'", specifier = ">=6" }, { name = "rich", marker = "extra == 'all'", specifier = ">=12,<14" }, { name = "rich", marker = "extra == 'ctl'", specifier = ">=12,<14" }, + { name = "ruamel-yaml", marker = "extra == 'all'", specifier = ">=0.18" }, + { name = "ruamel-yaml", marker = "extra == 'ctl'", specifier = ">=0.18" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=1.1.0" }, { name = "typer", marker = "extra == 'all'", specifier = ">=0.15.0" }, { name = "typer", marker = "extra == 'ctl'", specifier = ">=0.15.0" }, @@ -889,17 +893,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/31/10ac88f3357fc276dc8a64e8880c82e80e7459326ae1d0a211b40abf6665/ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216", size = 5606088, upload-time = "2025-05-31T16:39:09.613Z" } wheels = [ @@ -916,17 +920,17 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/34/29b18c62e39ee2f7a6a3bba7efd952729d8aadd45ca17efc34453b717665/ipython-9.6.0.tar.gz", hash = "sha256:5603d6d5d356378be5043e69441a072b50a5b33b4503428c77b04cb8ce7bc731", size = 4396932, upload-time = "2025-09-29T10:55:53.948Z" } wheels = [ @@ -938,7 +942,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1471,8 +1475,8 @@ name = "pendulum" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } wheels = [