Add error handling for executor deserialization in dgxcloud scheduler…

.coveragerc

@@ -0,0 +1,8 @@
+[paths]
+source =
+    nemo_run/
+    /home/runner/work/NeMo-Run/NeMo-Run/nemo_run
+    /opt/NeMo-Run/nemo_run
+[run]
+omit =
+    nemo_run/scripts/*

.github/workflows/build-test-publish-wheel.yml

@@ -0,0 +1,41 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build, test, and publish a PyPi wheel (to testpypi)
+
+on:
+  push:
+    branches:
+      - "main"
+      - "[rv][0-9].[0-9].[0-9]"
+      - "[rv][0-9].[0-9].[0-9]rc[0-9]"
+  pull_request:
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+jobs:
+  build-test-publish-wheel:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.23.0
+    with:
+      dry-run: true
+      python-package: nemo_run
+      python-version: "3.10"
+      packaging: hatch
+    secrets:
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

.github/workflows/cherry-pick-release-commit.yml

@@ -0,0 +1,14 @@
+name: Create PR to main with cherry-pick from release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  cherry-pick:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.22.7
+    secrets:
+      PAT: ${{ secrets.PAT }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/release-freeze.yml

@@ -0,0 +1,33 @@
+name: "Code freeze"
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-type:
+        type: choice
+        description: Type of release
+        options:
+          - major
+          - minor
+      freeze-commit:
+        type: string
+        description: Commit SHA to use for cut-off
+        required: false
+        default: main
+      dry-run:
+        type: boolean
+        description: Dry-run of code-freeze
+        required: false
+        default: true
+jobs:
+  code-freeze:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_code_freeze.yml@v0.22.5
+    with:
+      library-name: NeMo Run
+      python-package: nemo_run
+      release-type: ${{ inputs.release-type }}
+      freeze-commit: ${{ inputs.freeze-commit }}
+      dry-run: ${{ inputs.dry-run }}
+    secrets:
+      SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

.github/workflows/release.yml

@@ -0,0 +1,47 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: "Release NeMo Run"
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-ref:
+        description: Ref (SHA or branch name) to release
+        required: true
+        type: string
+      dry-run:
+        description: Do not publish a wheel and GitHub release.
+        required: true
+        default: true
+        type: boolean
+      version-bump-branch:
+        type: string
+        required: true
+        description: Branch to target for version bump
+jobs:
+  release:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_release_library.yml@v0.22.6
+    with:
+      release-ref: ${{ inputs.release-ref }}
+      python-package: nemo_run
+      python-version: "3.10"
+      library-name: NeMo Run
+      dry-run: ${{ inputs.dry-run }}
+      version-bump-branch: ${{ inputs.version-bump-branch }}
+    secrets:
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
+      PAT: ${{ secrets.PAT }}

.github/workflows/test.yml

@@ -27,17 +27,30 @@ jobs:
           git config --global user.name github-actions[bot]
           git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com
 
-      - name: Run tests
-        run: uv run -- pytest test/
-
       - name: Run coverage
         run: |
-          uv run -- coverage run -m pytest
+          uv run -- coverage run --branch --source=nemo_run -a -m pytest
           uv run -- coverage report -m
 
       - name: Run example notebooks
-        run: uv run -- jupyter execute examples/**/*.ipynb
+        run: |
+          uv run -- coverage run --branch --source=nemo_run -a -m jupyter execute examples/**/*.ipynb
+          uv run -- coverage report -m
 
       - name: Run example scripts
         run: |
-          cd examples/hello-world && uv run -- python hello_scripts.py
+          uv run -- coverage run --branch --source=nemo_run -a examples/hello-world/hello_scripts.py
+          uv run -- coverage report -m
+
+      - name: Generate report
+        run: |
+          uv run -- coverage xml
+          uv run -- coverage report
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          verbose: true
+          flags: cpu

codecov.yml

@@ -0,0 +1,7 @@
+comment: false
+coverage:
+  status:
+    patch: false
+    project: false
+fixes:
+  - "/opt/NeMo-Run/::"

nemo_run/__init__.py

@@ -13,14 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nemo_run import cli
 from nemo_run.api import autoconvert, dryrun_fn
 from nemo_run.config import Config, ConfigurableMixin, Partial, Script
-from nemo_run.core.execution.base import (
-    Executor,
-    ExecutorMacros,
-    import_executor,
-)
+from nemo_run.core.execution.base import Executor, ExecutorMacros, import_executor
 from nemo_run.core.execution.dgxcloud import DGXCloudExecutor
 from nemo_run.core.execution.docker import DockerExecutor
 from nemo_run.core.execution.launcher import FaultTolerance, SlurmRay, SlurmTemplate, Torchrun
@@ -32,10 +27,13 @@
 from nemo_run.devspace.base import DevSpace
 from nemo_run.help import help
 from nemo_run.lazy import LazyEntrypoint, lazy_imports
+from nemo_run.package_info import __package_name__, __version__
 from nemo_run.run.api import run
 from nemo_run.run.experiment import Experiment
 from nemo_run.run.plugin import ExperimentPlugin as Plugin
 
+from nemo_run import cli
+
 __all__ = [
     "autoconvert",
     "cli",
@@ -70,9 +68,6 @@
     "Torchrun",
     "SlurmRay",
     "SlurmTemplate",
+    "__version__",
+    "__package_name__",
 ]
-
-try:
-    from nemo_run._version import __version__
-except Exception:
-    __version__ = "0.0.1"

nemo_run/config.py

@@ -118,9 +118,9 @@ def from_dict(raw_data: dict | list | str | float | int | bool, cls: Type[_T]) -
     if isinstance(raw_data, dict):
         underlying_types = get_underlying_types(cls)
         underlying_types = [tp for tp in underlying_types if tp is not type(None)]
-        assert (
-            len(underlying_types) == 1
-        ), f"Unable to load {cls}. Nested union types are not currently supported."
+        assert len(underlying_types) == 1, (
+            f"Unable to load {cls}. Nested union types are not currently supported."
+        )
         cls = underlying_types[0]  # type: ignore
 
     if dataclasses.is_dataclass(cls):

nemo_run/core/execution/base.py

@@ -110,9 +110,9 @@ def get_launcher(self) -> Launcher:
             self._setup_launcher()
             self._launcher_setup = True
 
-        assert self.launcher is None or isinstance(
-            self.launcher, Launcher
-        ), f"{self.info()} could not setup the launcher."
+        assert self.launcher is None or isinstance(self.launcher, Launcher), (
+            f"{self.info()} could not setup the launcher."
+        )
         if self.launcher is None:
             self.launcher = Launcher()
 

nemo_run/core/execution/dgxcloud.py

@@ -234,7 +234,9 @@ def assign(
                 == os.path.abspath(x["path"]),
                 self.pvcs,
             )
-        ), f"Need to specify atleast one PVC containing {self.job_dir}.\nTo update job dir to a PVC path, you can use set_nemorun_home() or the NEMORUN_HOME env var."
+        ), (
+            f"Need to specify atleast one PVC containing {self.job_dir}.\nTo update job dir to a PVC path, you can use set_nemorun_home() or the NEMORUN_HOME env var."
+        )
 
     def package(self, packager: Packager, job_name: str):
         assert self.experiment_id, "Executor not assigned to an experiment."

nemo_run/core/execution/skypilot.py

@@ -107,12 +107,12 @@ class SkypilotExecutor(Executor):
     packager: Packager = field(default_factory=lambda: GitArchivePackager())  # type: ignore  # noqa: F821
 
     def __post_init__(self):
-        assert (
-            _SKYPILOT_AVAILABLE
-        ), "Skypilot is not installed. Please install it using `pip install nemo_run[skypilot]"
-        assert isinstance(
-            self.packager, GitArchivePackager
-        ), "Only GitArchivePackager is currently supported for SkypilotExecutor."
+        assert _SKYPILOT_AVAILABLE, (
+            "Skypilot is not installed. Please install it using `pip install nemo_run[skypilot]"
+        )
+        assert isinstance(self.packager, GitArchivePackager), (
+            "Only GitArchivePackager is currently supported for SkypilotExecutor."
+        )
 
     @classmethod
     def parse_app(cls: Type["SkypilotExecutor"], app_id: str) -> tuple[str, str, int]: