diff --git a/nemo_run/run/logs.py b/nemo_run/run/logs.py index cda7c6f9..6c205348 100644 --- a/nemo_run/run/logs.py +++ b/nemo_run/run/logs.py @@ -21,12 +21,17 @@ from typing import Optional, TextIO from torchx.cli.cmd_log import _prefix_line, find_role_replicas, validate -from torchx.cli.colors import ENDC, GREEN from torchx.schedulers.api import Stream from torchx.specs.api import is_started, parse_app_handle from torchx.specs.builders import make_app_handle from torchx.util.types import none_throws +# TorchX moved color constants from `torchx.cli.colors` to `torchx.util.colors` on main. +try: + from torchx.util.colors import ENDC, GREEN +except ImportError: # pragma: no cover - compatibility with older TorchX releases + from torchx.cli.colors import ENDC, GREEN + from nemo_run.core.execution.base import LogSupportedExecutor from nemo_run.core.frontend.console.api import CONSOLE from nemo_run.run.torchx_backend.runner import Runner, get_runner diff --git a/nemo_run/run/torchx_backend/runner.py b/nemo_run/run/torchx_backend/runner.py index 7de27e83..4fd11362 100644 --- a/nemo_run/run/torchx_backend/runner.py +++ b/nemo_run/run/torchx_backend/runner.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import logging from typing import Any, Optional @@ -63,7 +64,12 @@ def dryrun( # type: ignore with log_event("dryrun", scheduler): sched = self._scheduler(scheduler) - sched._validate(app, scheduler) + # TorchX `main` added `cfg` to `_validate`; older versions only accept + # `(app, scheduler)`. Support both signatures. + if "cfg" in inspect.signature(sched._validate).parameters: + sched._validate(app, scheduler, cfg) + else: + sched._validate(app, scheduler) dryrun_info = sched.submit_dryrun(app, cfg) dryrun_info._scheduler = scheduler return dryrun_info diff --git a/pyproject.toml b/pyproject.toml index b90260a5..11b4d16f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,16 +23,18 @@ dependencies = [ "catalogue>=2.0.10", "fabric>=3.2.2", "fiddle>=0.3.0", - "torchx>=0.7.0", + "torchx @ git+https://github.com/meta-pytorch/torchx.git@main", "typer>=0.12.3", "rich>=13.7.1", "jinja2>=3.1.4", - "cryptography < 43.0.0", + "importlib-metadata", + "cryptography", "networkx >= 3.3", "omegaconf>=2.3.0", "leptonai>=0.26.6", "packaging", "toml", + "urllib3>=2.6.3", ] readme = "README.md" requires-python = ">= 3.10" diff --git a/uv.lock b/uv.lock index 13e2a256..9580a85f 100644 --- a/uv.lock +++ b/uv.lock @@ -3231,7 +3231,7 @@ wheels = [ [[package]] name = "ibm-cloud-sdk-core" -version = "3.18.0" +version = "3.24.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyjwt" }, @@ -3239,39 +3239,41 @@ dependencies = [ { name = "requests" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/82/5fa378337357b0cdcc1f290b6b78457549939054e24f32e54ad378e0d663/ibm-cloud-sdk-core-3.18.0.tar.gz", hash = "sha256:bf2b6940761970c14b53fc8fc135af567c58b9709db3214be403a383dd51cabb", size = 57929, upload-time = "2023-11-15T18:07:51.467Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/e5/5de9646a57209bd59ee504ea11b9d83fab0e273c648d5cb9ec529c2dfb21/ibm_cloud_sdk_core-3.24.1.tar.gz", hash = "sha256:96b4b4deefc7b72874ef44545362e63ee3a7d6a4c82e68f36c482c8033683ab3", size = 76252, upload-time = "2025-05-30T15:47:12.802Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/a9/7356d14186089da1290a77d9e6e737f4cb62ab75e0cbe405ba6be7d3b322/ibm_cloud_sdk_core-3.24.1-py3-none-any.whl", hash = "sha256:28eb70379977bac15318a07634e4b56d9de671186090d8accec180b77ffb7969", size = 75783, upload-time = "2025-05-30T15:47:11.686Z" }, +] [[package]] name = "ibm-cos-sdk" -version = "2.13.5" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ibm-cos-sdk-core" }, { name = "ibm-cos-sdk-s3transfer" }, { name = "jmespath" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/e6/908791df6fd31d94e80be9597242eac2b942c28b0d4ea1fc6b963d4a6be3/ibm-cos-sdk-2.13.5.tar.gz", hash = "sha256:1aff7f9863ac9072a3db2f0053bec99478b26f3fb5fa797ce96a15bbb13cd40e", size = 58638, upload-time = "2024-06-11T16:39:10.253Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/ca/0d9856508d93dd66086286a66fb368cd69f6c90dfe1e5116713cec2e2866/ibm-cos-sdk-2.10.0.tar.gz", hash = "sha256:053748dee2920129bc876e4740120b3a89ab21591c41113b318ab198866b1fd8", size = 52590, upload-time = "2021-03-15T21:21:03.37Z" } [[package]] name = "ibm-cos-sdk-core" -version = "2.13.5" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "requests" }, - { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/ff/cabdf3d7892c3607297ffb7d77c3caafe2f50f876d29e90c44ac67495cef/ibm-cos-sdk-core-2.13.5.tar.gz", hash = "sha256:d3a99d8b06b3f8c00b1a9501f85538d592463e63ddf8cec32672ab5a0b107b83", size = 1101815, upload-time = "2024-06-11T16:38:58.085Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/66/c51ccc8e318de67aa738aec3560ee93164d7547addac5f59e56b1f293205/ibm-cos-sdk-core-2.10.0.tar.gz", hash = "sha256:96c6554cb829d7d84d5bb0f74770bff38bae18ef28eb950e31849fb70cd33576", size = 888166, upload-time = "2021-03-15T21:20:51.634Z" } [[package]] name = "ibm-cos-sdk-s3transfer" -version = "2.13.5" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ibm-cos-sdk-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/42/f2faefd0c3360928336b93a14db2aff25f556aa50252188efd1ba363e371/ibm-cos-sdk-s3transfer-2.13.5.tar.gz", hash = "sha256:9649b1f2201c6de96ff5a6b5a3686de3a809e6ef3b8b12c7c4f2f7ce72da7749", size = 139491, upload-time = "2024-06-11T16:39:04.148Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/12/437a2dbad65815e207b526f7d7e0ce5bfe8f239796be1661feeb4c9218c9/ibm-cos-sdk-s3transfer-2.10.0.tar.gz", hash = "sha256:8d679ee0d2b9ecfadc08245eb5404dba2351b1ab8514004ba147760cb8102270", size = 133248, upload-time = "2021-03-15T21:20:57.286Z" } [[package]] name = "ibm-platform-services" @@ -3494,11 +3496,11 @@ wheels = [ [[package]] name = "jmespath" -version = "1.0.1" +version = "0.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/56/3f325b1eef9791759784aa5046a8f6a1aff8f7c898a2e34506771d3b99d8/jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", size = 21607, upload-time = "2020-05-12T22:03:47.267Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, + { url = "https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f", size = 24489, upload-time = "2020-05-12T22:03:45.643Z" }, ] [[package]] @@ -4306,15 +4308,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/b7/b9e70fde2c0f0c9af4cc5277782a89b66d35948ea3369ec9f598358c3ac5/multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506", size = 10051, upload-time = "2024-09-09T23:49:36.506Z" }, ] -[[package]] -name = "mypy-extensions" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433, upload-time = "2023-02-04T12:11:27.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695, upload-time = "2023-02-04T12:11:25.002Z" }, -] - [[package]] name = "myst-parser" version = "4.0.1" @@ -4416,6 +4409,7 @@ dependencies = [ { name = "cryptography" }, { name = "fabric" }, { name = "fiddle" }, + { name = "importlib-metadata" }, { name = "inquirerpy" }, { name = "jinja2" }, { name = "leptonai" }, @@ -4426,6 +4420,7 @@ dependencies = [ { name = "toml" }, { name = "torchx" }, { name = "typer" }, + { name = "urllib3" }, ] [package.optional-dependencies] @@ -4472,9 +4467,10 @@ lint = [ [package.metadata] requires-dist = [ { name = "catalogue", specifier = ">=2.0.10" }, - { name = "cryptography", specifier = "<43.0.0" }, + { name = "cryptography" }, { name = "fabric", specifier = ">=3.2.2" }, { name = "fiddle", specifier = ">=0.3.0" }, + { name = "importlib-metadata" }, { name = "inquirerpy", specifier = ">=0.3.4" }, { name = "jinja2", specifier = ">=3.1.4" }, { name = "kubernetes", marker = "extra == 'ray'" }, @@ -4487,8 +4483,9 @@ requires-dist = [ { name = "skypilot", extras = ["all"], marker = "extra == 'skypilot-all'", specifier = ">=0.10.0" }, { name = "skypilot", extras = ["kubernetes"], marker = "extra == 'skypilot'", specifier = ">=0.10.0" }, { name = "toml" }, - { name = "torchx", specifier = ">=0.7.0" }, + { name = "torchx", git = "https://github.com/meta-pytorch/torchx.git?rev=main" }, { name = "typer", specifier = ">=0.12.3" }, + { name = "urllib3", specifier = ">=2.6.3" }, ] provides-extras = ["ray", "skypilot", "skypilot-all"] @@ -5807,19 +5804,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/a7/c8a2d361bf89c0d9577c934ebb7421b25dc84bf3a8e3ac0a40aed9acc547/pyparsing-3.2.1-py3-none-any.whl", hash = "sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1", size = 107716, upload-time = "2024-12-31T20:59:42.738Z" }, ] -[[package]] -name = "pyre-extensions" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/53/5bc2532536e921c48366ad1047c1344ccef6afa5e84053f0f6e20a453767/pyre_extensions-0.0.32.tar.gz", hash = "sha256:5396715f14ea56c4d5fd0a88c57ca7e44faa468f905909edd7de4ad90ed85e55", size = 10852, upload-time = "2024-11-22T19:26:44.152Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/7a/9812cb8be9828ab688203c5ac5f743c60652887f0c00995a6f6f19f912bd/pyre_extensions-0.0.32-py3-none-any.whl", hash = "sha256:a63ba6883ab02f4b1a9f372ed4eb4a2f4c6f3d74879aa2725186fdfcfe3e5c68", size = 12766, upload-time = "2024-11-22T19:26:42.465Z" }, -] - [[package]] name = "pyreadline3" version = "3.5.4" @@ -7211,22 +7195,16 @@ wheels = [ [[package]] name = "torchx" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } +version = "0.8.0.dev0" +source = { git = "https://github.com/meta-pytorch/torchx.git?rev=main#029982c43d3b94a013be322d78d5dd2246542fe0" } dependencies = [ { name = "docker" }, { name = "docstring-parser" }, { name = "filelock" }, { name = "fsspec" }, - { name = "importlib-metadata" }, - { name = "pyre-extensions" }, { name = "pyyaml" }, { name = "tabulate" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/a9/e07e64222e59f44eb18b333bc8db942a295c0d9c16f04c8703fe106a8fd5/torchx-0.7.0.tar.gz", hash = "sha256:933b800849c69ddff9feda931f0ae2c4083638eafc73abfdf361158667c68ad6", size = 196142, upload-time = "2024-07-16T22:06:57.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/8d/282b884cef8f26b240e1ac8e60513a25a68e3463dc1c68771191a2c8a209/torchx-0.7.0-py3-none-any.whl", hash = "sha256:815c2628c30de1f5938c14c4427cfe280fdf93473b171be906ab7c1ea5971824", size = 256100, upload-time = "2024-07-16T21:16:01.849Z" }, + { name = "typing-extensions" }, ] [[package]] @@ -7314,19 +7292,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438, upload-time = "2024-06-07T18:52:13.582Z" }, ] -[[package]] -name = "typing-inspect" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, -] - [[package]] name = "typing-inspection" version = "0.4.0" @@ -7422,11 +7387,11 @@ wheels = [ [[package]] name = "urllib3" -version = "1.26.20" +version = "2.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" }, + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] [[package]]