diff --git a/.agents/skills/update-project-version/SKILL.md b/.agents/skills/update-project-version/SKILL.md new file mode 100644 index 000000000..3bfe9a22e --- /dev/null +++ b/.agents/skills/update-project-version/SKILL.md @@ -0,0 +1,109 @@ +--- +name: update-project-version +description: Update the NeMo Fabric release version across Cargo, setuptools package metadata, internal Python dependency pins, integration metadata, and lockfiles. Use when bumping, synchronizing, or auditing Fabric package versions for a release. +author: NVIDIA Corporation and Affiliates +license: Apache-2.0 +--- + + +# Update Project Version + +## Companion Guidance + +Use `karpathy-guidelines` alongside this skill for implementation or review +work. Keep changes scoped, surface assumptions, and define focused validation +before editing. + +Use this skill when changing the NeMo Fabric version, including +pre-release or build-metadata variants used during packaging. + +## Source Of Truth + +- `Cargo.toml` `[workspace.package].version` is the source of truth for the Rust + workspace and Python build versioning. +- Keep `Cargo.toml` `[workspace.dependencies]` self-references aligned when the + workspace version changes. +- `python/pyproject.toml` is the exception among the Python projects: do not add + a literal `project.version`. Keep `project.dynamic = ["version"]` because + Maturin derives `nemo-fabric-runtime`'s version from + `crates/fabric-python/Cargo.toml`, which inherits the workspace version. +- The setuptools projects do not derive their versions from Cargo. Update the + literal `project.version` in every one of these files: + - `pyproject.toml` + - `adapters/**/pyproject.toml` +- Keep internal Python package requirement pins aligned with the Python release + version: + - All `nemo-fabric-* == ` requirements in the root + `pyproject.toml` optional dependencies. + - Each adapter's `nemo-fabric-adapters-common == ` dependency. + +For a normal release, use the same `X.Y.Z` string everywhere. For a prerelease +or build-metadata version, use valid Cargo SemVer in `Cargo.toml` and the +equivalent PEP 440 version in explicit Python metadata. Confirm that the +Maturin-built runtime and setuptools-built packages resolve to equivalent +versions rather than blindly copying incompatible syntax. + +## Workflow + +1. Read the current version from `Cargo.toml` and decide the exact Cargo and + Python target version strings. +2. Run `just set-version `. The recipe converts supported Cargo + SemVer prereleases to PEP 440 and updates: + - `Cargo.toml` `[workspace.package].version` + - `Cargo.toml` `workspace.dependencies.fabric-core.version` + - The root setuptools `project.version` and every + `adapters/**/pyproject.toml` `project.version` + - Every internal `nemo-fabric-*` exact-version requirement + - `Cargo.lock` through Cargo metadata resolution + - The root, runtime, and adapter `uv.lock` files through `just lock-python` +3. Confirm that `python/pyproject.toml` remains dynamic and unchanged. +4. Audit references to the old version with targeted searches. Distinguish + package-version surfaces from examples and unrelated dependency versions. + +If editing the helper code, keep these contracts aligned: + +- `set_project_version` must call the Cargo and Python project version helpers. +- `set_cargo_workspace_version` must update the workspace version and the + `fabric-core` workspace dependency, then verify every `fabric-*` workspace + package through Cargo metadata. +- `set_python_project_versions` must update the root setuptools version, every + adapter `pyproject.toml` discovered recursively under `adapters/`, and all + internal exact-version pins while rejecting a static version in + `python/pyproject.toml`. +- The `set-version` recipe must run `just lock-python` after source metadata is + updated. + +## Validation + +- Inspect Cargo version fields: + `rg -n '^version =|fabric-core = \{ path = .*version =' Cargo.toml` +- Inspect explicit Python versions and internal pins: + `rg -n '^version =|nemo-fabric-[a-z-]+ == ' pyproject.toml adapters --glob 'pyproject.toml'` +- Confirm the runtime remains dynamic: + `rg -n 'dynamic = \["version"\]' python/pyproject.toml` +- Run `cargo check --workspace --locked`. +- Run `just build-python` to verify all Python package metadata resolves. +- Run `just test-python` when the integration version or Python packaging + behavior changes materially. +- Run `just wheels` for release-facing validation of every Python wheel. +- Run `git diff --check`. + +## Avoid + +- Updating only `Cargo.toml` and leaving the setuptools packages stale. +- Adding a literal version to `python/pyproject.toml`; Maturin owns that version. +- Updating Python package versions without their exact internal dependency pins. +- Forgetting `Cargo.lock`, the root `uv.lock`, or per-project `uv.lock` files. +- Blind repository-wide replacement of version-like strings. + +## References + +- `Cargo.toml` +- `Cargo.lock` +- `pyproject.toml` +- `uv.lock` +- `python/pyproject.toml` +- `python/uv.lock` +- `adapters/**/pyproject.toml` +- `adapters/**/uv.lock` +- `justfile` diff --git a/adapters/codex-cli/pyproject.toml b/adapters/codex-cli/pyproject.toml index 23bdb5c8d..6ae23358c 100644 --- a/adapters/codex-cli/pyproject.toml +++ b/adapters/codex-cli/pyproject.toml @@ -4,17 +4,19 @@ [build-system] requires = [ "setuptools>=64", - "setuptools-scm>=8", - "setuptools_dynamic_dependencies>=1.0.0", ] build-backend = "setuptools.build_meta" [project] name = "nemo-fabric-adapters-codex-cli" +version = "0.1.0" description = "Codex CLI adapter for NeMo Fabric" readme = "README.md" requires-python = ">=3.11" -dynamic = ["version", "dependencies"] +dependencies = [ + "nemo-fabric-adapters-common == 0.1.0", + "tomli-w~=1.2", +] [tool.setuptools.packages.find] where = ["src"] @@ -23,15 +25,5 @@ include = ["nemo_fabric_adapters.codex_cli*"] [tool.setuptools.data-files] "share/nemo-fabric/adapters/codex-cli" = ["fabric-adapter.json"] -[tool.setuptools_dynamic_dependencies] -dependencies = [ - "nemo-fabric-adapters-common == {version}", - "tomli-w~=1.2", -] - [tool.uv.sources] nemo-fabric-adapters-common = { path = "../common" } - -[tool.setuptools_scm] -root = "../.." -git_describe_command = "git describe --long --first-parent" diff --git a/adapters/codex-cli/uv.lock b/adapters/codex-cli/uv.lock index 4ba066c66..c2125e62a 100644 --- a/adapters/codex-cli/uv.lock +++ b/adapters/codex-cli/uv.lock @@ -4,6 +4,7 @@ requires-python = ">=3.11" [[package]] name = "nemo-fabric-adapters-codex-cli" +version = "0.1.0" source = { editable = "." } dependencies = [ { name = "nemo-fabric-adapters-common" }, @@ -18,6 +19,7 @@ requires-dist = [ [[package]] name = "nemo-fabric-adapters-common" +version = "0.1.0" source = { directory = "../common" } [[package]] diff --git a/adapters/common/pyproject.toml b/adapters/common/pyproject.toml index 993b0b6c6..77566d1ec 100644 --- a/adapters/common/pyproject.toml +++ b/adapters/common/pyproject.toml @@ -4,21 +4,15 @@ [build-system] requires = [ "setuptools>=64", - "setuptools-scm>=8", - "setuptools_dynamic_dependencies>=1.0.0", ] build-backend = "setuptools.build_meta" [project] name = "nemo-fabric-adapters-common" +version = "0.1.0" description = "Shared Python helpers for NeMo Fabric adapters" requires-python = ">=3.11" -dynamic = ["version"] [tool.setuptools.packages.find] where = ["src"] include = ["nemo_fabric_adapters.common*"] - -[tool.setuptools_scm] -root = "../.." -git_describe_command = "git describe --long --first-parent" diff --git a/adapters/common/uv.lock b/adapters/common/uv.lock index 224fc1a3f..a9e754d66 100644 --- a/adapters/common/uv.lock +++ b/adapters/common/uv.lock @@ -4,4 +4,5 @@ requires-python = ">=3.11" [[package]] name = "nemo-fabric-adapters-common" +version = "0.1.0" source = { editable = "." } diff --git a/adapters/hermes-cli/pyproject.toml b/adapters/hermes-cli/pyproject.toml index d61e6f859..762f71537 100644 --- a/adapters/hermes-cli/pyproject.toml +++ b/adapters/hermes-cli/pyproject.toml @@ -4,17 +4,20 @@ [build-system] requires = [ "setuptools>=64", - "setuptools-scm>=8", - "setuptools_dynamic_dependencies>=1.0.0", ] build-backend = "setuptools.build_meta" [project] name = "nemo-fabric-adapters-hermes-cli" +version = "0.1.0" description = "Hermes CLI adapter for NeMo Fabric" readme = "README.md" requires-python = ">=3.11" -dynamic = ["version", "dependencies"] +dependencies = [ + "nemo-fabric-adapters-common == 0.1.0", + "pyyaml>=6.0", # needed for writing hermes config files + "tomli-w~=1.2", # Needed by adapters to write relay config files +] [tool.setuptools.packages.find] where = ["src"] @@ -23,16 +26,5 @@ include = ["nemo_fabric_adapters.hermes_cli*"] [tool.setuptools.data-files] "share/nemo-fabric/adapters/hermes-cli" = ["fabric-adapter.json"] -[tool.setuptools_dynamic_dependencies] -dependencies = [ - "nemo-fabric-adapters-common == {version}", - "pyyaml>=6.0", # needed for writing hermes config files - "tomli-w~=1.2", # Needed by adapters to write relay config files -] - [tool.uv.sources] nemo-fabric-adapters-common = { path = "../common" } - -[tool.setuptools_scm] -root = "../.." -git_describe_command = "git describe --long --first-parent" diff --git a/adapters/hermes-cli/uv.lock b/adapters/hermes-cli/uv.lock index 886af6cc9..4bc4019f7 100644 --- a/adapters/hermes-cli/uv.lock +++ b/adapters/hermes-cli/uv.lock @@ -4,10 +4,12 @@ requires-python = ">=3.11" [[package]] name = "nemo-fabric-adapters-common" +version = "0.1.0" source = { directory = "../common" } [[package]] name = "nemo-fabric-adapters-hermes-cli" +version = "0.1.0" source = { editable = "." } dependencies = [ { name = "nemo-fabric-adapters-common" }, diff --git a/adapters/hermes-sdk/pyproject.toml b/adapters/hermes-sdk/pyproject.toml index 9b4d49be8..4398b7e63 100644 --- a/adapters/hermes-sdk/pyproject.toml +++ b/adapters/hermes-sdk/pyproject.toml @@ -4,17 +4,19 @@ [build-system] requires = [ "setuptools>=64", - "setuptools-scm>=8", - "setuptools_dynamic_dependencies>=1.0.0", ] build-backend = "setuptools.build_meta" [project] name = "nemo-fabric-adapters-hermes-sdk" +version = "0.1.0" description = "Hermes SDK adapter for NeMo Fabric" readme = "README.md" requires-python = ">=3.11" -dynamic = ["version", "dependencies"] +dependencies = [ + "nemo-fabric-adapters-common == 0.1.0", + "pyyaml>=6.0", # needed for writing hermes config files +] [tool.setuptools.packages.find] where = ["src"] @@ -23,15 +25,5 @@ include = ["nemo_fabric_adapters.hermes_sdk*"] [tool.setuptools.data-files] "share/nemo-fabric/adapters/hermes-sdk" = ["fabric-adapter.json"] -[tool.setuptools_dynamic_dependencies] -dependencies = [ - "nemo-fabric-adapters-common == {version}", - "pyyaml>=6.0", # needed for writing hermes config files -] - [tool.uv.sources] nemo-fabric-adapters-common = { path = "../common" } - -[tool.setuptools_scm] -root = "../.." -git_describe_command = "git describe --long --first-parent" diff --git a/adapters/hermes-sdk/uv.lock b/adapters/hermes-sdk/uv.lock index ebce2a728..af5e18c9c 100644 --- a/adapters/hermes-sdk/uv.lock +++ b/adapters/hermes-sdk/uv.lock @@ -4,10 +4,12 @@ requires-python = ">=3.11" [[package]] name = "nemo-fabric-adapters-common" +version = "0.1.0" source = { directory = "../common" } [[package]] name = "nemo-fabric-adapters-hermes-sdk" +version = "0.1.0" source = { editable = "." } dependencies = [ { name = "nemo-fabric-adapters-common" }, diff --git a/justfile b/justfile index a3871967a..83b3c3cd2 100644 --- a/justfile +++ b/justfile @@ -7,9 +7,241 @@ export REPO_ROOT := justfile_directory() # Skip dependency synchronization when the project environment is already fully synced. no_uv := "false" +# When set, versioning and packaging targets use this exact release version. +ref_name := "" python_projects := ". python adapters/common adapters/codex-cli adapters/hermes-cli adapters/hermes-sdk" +bash_helpers := ''' +set -euo pipefail + +uv_python_executable() { + ( + cd "$REPO_ROOT" + uv python find + ) +} + +semver_to_pep440() { + local python_executable="" + python_executable="$(uv_python_executable)" + + "$python_executable" - "$1" <<'PY' +import re +import sys + +pattern = re.compile( + r"^(?P\d+\.\d+\.\d+)" + r"(?:-(?Palpha|beta|rc)(?:\.(?P\d+))?)?" + r"(?:\+(?P[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" +) +match = pattern.fullmatch(sys.argv[1]) +if not match: + raise SystemExit( + "Unsupported package version format. Expected SemVer with optional " + "alpha/beta/rc prerelease and optional build metadata." + ) + +pep440 = match.group("release") +pre_label = match.group("pre_label") +if pre_label: + pre_map = {"alpha": "a", "beta": "b", "rc": "rc"} + pre_num = match.group("pre_num") or "0" + pep440 += f"{pre_map[pre_label]}{pre_num}" + +local = match.group("local") +if local: + normalized_local = ".".join( + part.lower() for part in re.split(r"[._-]+", local) if part + ) + if not normalized_local: + raise SystemExit("Python package local version metadata cannot be empty") + pep440 += f"+{normalized_local}" + +print(pep440) +PY +} + +set_cargo_workspace_version() { + local version="$1" + local python_executable="" + python_executable="$(uv_python_executable)" + + "$python_executable" - "$version" <<'PY' +from pathlib import Path +import re +import sys + +version = sys.argv[1] +if version.startswith("v"): + raise SystemExit("Release tags must not start with 'v'; use raw SemVer such as 0.1.0") +if not re.fullmatch( + r"\d+\.\d+\.\d+(?:-(?:alpha|beta|rc)(?:\.\d+)?)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?", + version, +): + raise SystemExit( + f"Unsupported release version '{version}'; use 0.1.0 or a supported " + "prerelease such as 0.1.0-rc.1" + ) + +path = Path("Cargo.toml") +text = path.read_text() +section = "" +output = [] +changed = [] +found_workspace_version = False +found_fabric_core = False + +for line in text.splitlines(keepends=True): + section_match = re.match(r"^\s*\[([^\]]+)\]\s*(?:#.*)?$", line) + if section_match: + section = section_match.group(1) + + updated = line + if section == "workspace.package": + updated, count = re.subn( + r'^(version\s*=\s*")([^"]+)(".*)$', + rf"\g<1>{version}\g<3>", + line, + ) + if count == 1: + found_workspace_version = True + if updated != line: + changed.append("workspace.package.version") + elif section == "workspace.dependencies": + updated, count = re.subn( + r'^(fabric-core\s*=\s*\{[^}]*\bversion\s*=\s*")([^"]+)(".*)$', + rf"\g<1>{version}\g<3>", + line, + ) + if count == 1: + found_fabric_core = True + if updated != line: + changed.append("workspace.dependencies.fabric-core.version") + + output.append(updated) + +missing = [] +if not found_workspace_version: + missing.append("workspace.package.version") +if not found_fabric_core: + missing.append("workspace.dependencies.fabric-core.version") +if missing: + raise SystemExit(f"Failed to find expected Cargo version fields: {', '.join(missing)}") + +path.write_text("".join(output)) +if changed: + print(f"Cargo.toml version set to {version}: {', '.join(changed)}") +else: + print(f"Cargo.toml already set to {version}") +PY + + local metadata_file="" + metadata_file="$(mktemp)" + if ! cargo metadata --no-deps --format-version 1 > "$metadata_file"; then + rm -f "$metadata_file" + return 1 + fi + if ! "$python_executable" - "$version" "$metadata_file" <<'PY' +import json +import sys +from pathlib import Path + +version = sys.argv[1] +metadata = json.loads(Path(sys.argv[2]).read_text()) +workspace_members = set(metadata["workspace_members"]) +mismatched = [] +checked = 0 + +for package in metadata["packages"]: + if package["id"] not in workspace_members or not package["name"].startswith("fabric-"): + continue + checked += 1 + if package["version"] != version: + mismatched.append(f"{package['name']}={package['version']}") + +if checked == 0: + raise SystemExit("Cargo metadata did not include any Fabric workspace packages") +if mismatched: + raise SystemExit( + f"Cargo workspace packages do not all resolve to {version}: {', '.join(mismatched)}" + ) +print(f"Cargo metadata resolves {checked} Fabric workspace packages to {version}") +PY + then + rm -f "$metadata_file" + return 1 + fi + rm -f "$metadata_file" +} + +set_python_project_versions() { + local version="" + local python_executable="" + version="$(semver_to_pep440 "$1")" + python_executable="$(uv_python_executable)" + + "$python_executable" - "$version" <<'PY' +from pathlib import Path +import re +import sys +import tomllib + +version = sys.argv[1] +project_paths = ( + Path("pyproject.toml"), + *sorted(Path("adapters").glob("**/pyproject.toml")), +) +pin_pattern = re.compile(r'(nemo-fabric-[a-z0-9-]+\s*==\s*)([^"\s,]+)') + +for path in project_paths: + text = path.read_text() + updated, count = re.subn( + r'^version\s*=\s*"[^"]+"$', + f'version = "{version}"', + text, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise SystemExit(f"Failed to find exactly one project version in {path}") + updated = pin_pattern.sub(rf"\g<1>{version}", updated) + if updated != text: + path.write_text(updated) + print(f"{path} version and internal pins updated to {version}") + else: + print(f"{path} already set to {version}") + +runtime_path = Path("python/pyproject.toml") +runtime = tomllib.loads(runtime_path.read_text()) +project = runtime.get("project", {}) +if "version" in project or "version" not in project.get("dynamic", []): + raise SystemExit( + "python/pyproject.toml must keep a dynamic version derived from Cargo.toml" + ) + +mismatched_pins = [] +for path in project_paths: + for match in pin_pattern.finditer(path.read_text()): + if match.group(2) != version: + mismatched_pins.append(f"{path}: {match.group(0)}") +if mismatched_pins: + raise SystemExit( + "Internal Python dependency pins are not synchronized: " + + ", ".join(mismatched_pins) + ) +print("python/pyproject.toml continues to derive its version from Cargo.toml") +PY +} + +set_project_version() { + local version="$1" + set_cargo_workspace_version "$version" + set_python_project_versions "$version" +} +''' + # Remove local Rust and Python build and test artifacts. clean: #!/usr/bin/env bash @@ -61,6 +293,22 @@ lock-python: uv lock --project "$project" done +# [version] or --set ref_name= +set-version version="": + #!/usr/bin/env bash + {{ bash_helpers }} + version="{{ version }}" + if [[ -z "$version" ]]; then + version="{{ ref_name }}" + fi + if [[ -z "$version" ]]; then + echo "Error: version is required for set-version" >&2 + exit 1 + fi + cd "$REPO_ROOT" + set_project_version "$version" + just lock-python + # Generate the Python and Rust API references and validate the Fern configuration. # --set [no_uv=true|false] docs: diff --git a/pyproject.toml b/pyproject.toml index d21fb62d4..79652eb62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,43 +4,38 @@ [build-system] requires = [ "setuptools>=64", - "setuptools-scm>=8", - "setuptools_dynamic_dependencies>=1.0.0", ] build-backend = "setuptools.build_meta" [project] name = "nemo-fabric" +version = "0.1.0" description = "Python SDK distribution for NeMo Fabric" requires-python = ">=3.11" -dynamic = ["version", "dependencies", "optional-dependencies"] +dependencies = [] [tool.setuptools] packages = [] -[tool.setuptools_dynamic_dependencies] -dependencies = [ -] - -[tool.setuptools_dynamic_dependencies.optional-dependencies] +[project.optional-dependencies] adapters-common = [ - "nemo-fabric-adapters-common == {version}", + "nemo-fabric-adapters-common == 0.1.0", ] adapters-codex-cli = [ - "nemo-fabric-adapters-codex-cli == {version}", + "nemo-fabric-adapters-codex-cli == 0.1.0", ] adapters-hermes-cli = [ - "nemo-fabric-adapters-hermes-cli == {version}", + "nemo-fabric-adapters-hermes-cli == 0.1.0", ] adapters-hermes-sdk = [ - "nemo-fabric-adapters-hermes-sdk == {version}", + "nemo-fabric-adapters-hermes-sdk == 0.1.0", ] codex = [ - "nemo-fabric-adapters-codex-cli == {version}", + "nemo-fabric-adapters-codex-cli == 0.1.0", ] harbor = [ @@ -49,8 +44,8 @@ harbor = [ ] hermes = [ - "nemo-fabric-adapters-hermes-cli == {version}", - "nemo-fabric-adapters-hermes-sdk == {version}", + "nemo-fabric-adapters-hermes-cli == 0.1.0", + "nemo-fabric-adapters-hermes-sdk == 0.1.0", "hermes-agent>=0.17.0", ] @@ -60,7 +55,7 @@ relay = [ ] runtime = [ - "nemo-fabric-runtime == {version}", + "nemo-fabric-runtime == 0.1.0", ] [dependency-groups] @@ -100,9 +95,6 @@ nemo-fabric-adapters-codex-cli = { path = "adapters/codex-cli" } nemo-fabric-adapters-hermes-cli = { path = "adapters/hermes-cli" } nemo-fabric-adapters-hermes-sdk = { path = "adapters/hermes-sdk" } -[tool.setuptools_scm] -git_describe_command = "git describe --long --first-parent" - [tool.pytest.ini_options] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" diff --git a/python/src/nemo_fabric/integrations/harbor/__init__.py b/python/src/nemo_fabric/integrations/harbor/__init__.py index 3c10cebd0..71c29dca0 100644 --- a/python/src/nemo_fabric/integrations/harbor/__init__.py +++ b/python/src/nemo_fabric/integrations/harbor/__init__.py @@ -5,6 +5,7 @@ from __future__ import annotations +import importlib.metadata import json import shlex import uuid @@ -53,7 +54,10 @@ def name() -> str: return "fabric" def version(self) -> str | None: - return "0.1.0" + try: + return importlib.metadata.version("nemo-fabric-runtime") + except importlib.metadata.PackageNotFoundError: + return None async def setup(self, environment: BaseEnvironment) -> None: result = await environment.exec("mkdir -p /logs/agent /tmp", timeout_sec=30) diff --git a/uv.lock b/uv.lock index d95e472e8..222ac640a 100644 --- a/uv.lock +++ b/uv.lock @@ -1591,6 +1591,7 @@ wheels = [ [[package]] name = "nemo-fabric" +version = "0.1.0" source = { editable = "." } [package.optional-dependencies] @@ -1692,6 +1693,7 @@ test = [ [[package]] name = "nemo-fabric-adapters-codex-cli" +version = "0.1.0" source = { directory = "adapters/codex-cli" } dependencies = [ { name = "nemo-fabric-adapters-common" }, @@ -1706,10 +1708,12 @@ requires-dist = [ [[package]] name = "nemo-fabric-adapters-common" +version = "0.1.0" source = { directory = "adapters/common" } [[package]] name = "nemo-fabric-adapters-hermes-cli" +version = "0.1.0" source = { directory = "adapters/hermes-cli" } dependencies = [ { name = "nemo-fabric-adapters-common" }, @@ -1726,6 +1730,7 @@ requires-dist = [ [[package]] name = "nemo-fabric-adapters-hermes-sdk" +version = "0.1.0" source = { directory = "adapters/hermes-sdk" } dependencies = [ { name = "nemo-fabric-adapters-common" },