Merge branch 'master' into autoscaler-ippr

Author: rueian

.buildkite/dependencies.rayci.yml

@@ -0,0 +1,28 @@
+group: dependencies
+depends_on:
+  - forge
+steps:
+  # dependencies
+  - label: ":tapioca: build: pip-compile dependencies"
+    key: pip_compile_dependencies
+    tags: always
+    instance_type: small
+    commands:
+      # uncomment the following line to update the pinned versions of pip dependencies
+      # to the latest versions; otherwise, the pinned versions will be re-used as much
+      # as possible
+      # - rm ./python/requirements_compiled.txt
+      - cp ./python/requirements_compiled.txt requirements_compiled_backup.txt
+      - ./ci/ci.sh compile_pip_dependencies
+      - cp -f ./python/requirements_compiled.txt /artifact-mount/
+      - diff ./python/requirements_compiled.txt requirements_compiled_backup.txt || (echo "requirements_compiled.txt is not up to date. Please download it from Artifacts tab and git push the changes." && exit 1)
+    job_env: oss-ci-base_test-py3.11
+    depends_on: oss-ci-base_test-multipy
+
+  - label: ":tapioca: build: raydepsets: compile LLM dependencies"
+    key: raydepsets_compile_llm_dependencies
+    tags: always
+    instance_type: small
+    command: ./ci/test_compile_llm_requirements.sh
+    job_env: oss-ci-base_test-py3.11
+    depends_on: oss-ci-base_test-multipy

.buildkite/others.rayci.yml

@@ -2,31 +2,6 @@ group: others
 depends_on:
   - forge
 steps:
-  # dependencies
-  - label: ":tapioca: build: pip-compile dependencies"
-    key: pip_compile_dependencies
-    tags: always
-    instance_type: small
-    commands:
-      # uncomment the following line to update the pinned versions of pip dependencies
-      # to the latest versions; otherwise, the pinned versions will be re-used as much
-      # as possible
-      # - rm ./python/requirements_compiled.txt
-      - cp ./python/requirements_compiled.txt requirements_compiled_backup.txt
-      - ./ci/ci.sh compile_pip_dependencies
-      - cp -f ./python/requirements_compiled.txt /artifact-mount/
-      - diff ./python/requirements_compiled.txt requirements_compiled_backup.txt || (echo "requirements_compiled.txt is not up to date. Please download it from Artifacts tab and git push the changes." && exit 1)
-    job_env: oss-ci-base_test-py3.11
-    depends_on: oss-ci-base_test-multipy
-
-  - label: ":tapioca: build: uv pip compile LLM dependencies"
-    key: uv_pip_compile_llm_dependencies
-    tags: always
-    instance_type: small
-    command: ./ci/test_compile_llm_requirements.sh
-    job_env: oss-ci-base_test-py3.11
-    depends_on: oss-ci-base_test-multipy
-
   # docs
   - name: doctestbuild
     wanda: ci/docker/doctest.build.wanda.yaml

BUILD.bazel

@@ -116,20 +116,8 @@ alias(
 # https://github.com/hedronvision/bazel-compile-commands-extractor?tab=readme-ov-file#vscode - directions for clangd config
 refresh_compile_commands(
     name = "refresh_compile_commands",
-    exclude_external_sources = True,  # removed below to have lsp index external cc files at the cost of 2x index time
-    # Specify the targets of interest.
-    # For example, specify a dict of targets and any flags required to build.
     targets = {
-        "//:ray_pkg_zip": "",
-    },
-    # No need to add flags already in .bazelrc. They're automatically picked up.
-)
-
-# bazel run :refresh_compile_commands_external_sources for generation with external source files (cc files)
-refresh_compile_commands(
-    name = "refresh_compile_commands_external_sources",
-    targets = {
-        "//:ray_pkg_zip": "",
+        "//:ray_pkg": "",
     },
 )
 
@@ -515,6 +503,18 @@ genrule(
     local = 1,
 )
 
+py_binary(
+    name = "gen_py_proto",
+    srcs = ["gen_py_proto.py"],
+    data = [
+        ":ray_py_proto_zip",
+    ],
+    visibility = ["//visibility:private"],
+    deps = [
+        "//bazel:gen_extract",
+    ],
+)
+
 py_binary(
     name = "gen_ray_pkg",
     srcs = ["gen_ray_pkg.py"],

bazel/gen_extract.py

@@ -16,7 +16,9 @@ def gen_extract(
 
     root_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY")
     if not root_dir:
-        raise ValueError("BUILD_WORKSPACE_DIRECTORY not set")
+        raise ValueError(
+            "BUILD_WORKSPACE_DIRECTORY not set; please run this script from 'bazelisk run'"
+        )
 
     if sub_dir:
         extract_dir = os.path.join(root_dir, sub_dir)

ci/lint/pydoclint-baseline.txt

@@ -307,7 +307,7 @@ python/ray/actor.py
     DOC201: Method `ActorMethod.options` does not have a return section in docstring
     DOC101: Method `_ActorClassMetadata.__init__`: Docstring contains fewer arguments than in function signature.
     DOC107: Method `_ActorClassMetadata.__init__`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints
-    DOC103: Method `_ActorClassMetadata.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [accelerator_type: , actor_creation_function_descriptor: , class_id: , concurrency_groups: , enable_tensor_transport: bool, label_selector: , language: , max_restarts: , max_task_retries: , memory: , modified_class: , num_cpus: , num_gpus: , object_store_memory: , resources: , runtime_env: , scheduling_strategy: SchedulingStrategyT].
+    DOC103: Method `_ActorClassMetadata.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [accelerator_type: , actor_creation_function_descriptor: , class_id: , concurrency_groups: , enable_tensor_transport: bool, label_selector: , language: , max_restarts: , max_task_retries: , memory: , method_meta: , modified_class: , num_cpus: , num_gpus: , object_store_memory: , resources: , runtime_env: , scheduling_strategy: SchedulingStrategyT].
     DOC101: Method `ActorClass.__init__`: Docstring contains fewer arguments than in function signature.
     DOC106: Method `ActorClass.__init__`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature
     DOC107: Method `ActorClass.__init__`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints

ci/pipeline/determine_tests_to_run.py

@@ -14,7 +14,7 @@
     lint python cpp core_cpp java workflow compiled_graphs dashboard ray_client
     data dask serve ml tune train llm rllib rllib_gpu rllib_directly
     linux_wheels macos_wheels docker doc python_dependencies tools
-    release_tests compiled_python spark_on_ray
+    release_tests compiled_python spark_on_ray runtime_env_container
     """.split()
 )
 

cpp/src/ray/runtime/runtime_env.cc

@@ -16,7 +16,7 @@
 #include <ray/api/runtime_env.h>
 #include <ray/util/logging.h>
 
-#include "src/ray/protobuf/runtime_env_common.pb.h"
+#include "src/ray/protobuf/public/runtime_environment.pb.h"
 
 namespace ray {
 

doc/source/data/images/dataset-shuffle.svg

@@ -436,7 +436,7 @@ We publish the dependencies that are installed in our ``ray`` Docker images for
     .. tab-item:: ray (Python 3.9)
         :sync: ray (Python 3.9)
 
-        Ray version: nightly (`f99d0ea <https://github.com/ray-project/ray/commit/f99d0ea7c4248f8d64eeb8f914d2742c91cd1948>`_)
+        Ray version: nightly (`ec5d410 <https://github.com/ray-project/ray/commit/ec5d4108737ead7dccb2930439842e0b2de012bb>`_)
 
         .. literalinclude:: ./pip_freeze_ray-py39-cpu.txt
 

doc/source/ray-overview/installation.rst

@@ -12,11 +12,12 @@ aiohappyeyeballs==2.6.1
 aiohttp==3.11.16
 aiohttp-cors==0.7.0
 aioitertools==0.11.0
-aiorwlock==1.5.0
+aiorwlock==1.3.0
 aiosignal==1.3.1
 aiosqlite==0.19.0
 ale-py==0.10.1
 alembic==1.12.1
+amqp==5.3.1
 annotated-types==0.6.0
 antlr4-python3-runtime==4.11.1
 anyio==3.7.1
@@ -32,11 +33,15 @@ astunparse==1.6.3
 async-timeout==4.0.3
 attrs==25.1.0
 ax-platform==0.3.2
+azure-common==1.1.28
+azure-core==1.29.5
+azure-storage-blob==12.22.0
 Babel==2.13.1
 backcall==0.2.0
 base58==2.0.1
 bayesian-optimization==1.4.3
 beautifulsoup4==4.11.1
+billiard==4.2.1
 bleach==6.1.0
 bokeh==2.4.3
 boltons @ file:///home/conda/feedstock_root/build_artifacts/boltons_1733827268945/work
@@ -45,12 +50,15 @@ boto3==1.26.76
 botocore==1.29.76
 botorch==0.8.5
 Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1749229842835/work
-build==1.2.2.post1
 cachetools==5.5.2
+celery==5.5.3
 certifi==2025.1.31
 cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1.2
+click-repl==0.3.0
 cloudpickle==2.2.0
 cma==3.2.2
 cmdstanpy==1.2.0
@@ -60,7 +68,7 @@ colorful==0.5.5
 colorlog==6.7.0
 comet-ml==3.44.1
 comm==0.2.0
-conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1749201703459/work/conda-src
+conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1754405245494/work/conda-src
 conda-libmamba-solver @ file:///home/conda/feedstock_root/build_artifacts/conda-libmamba-solver_1745834476052/work/src
 conda-package-handling @ file:///home/conda/feedstock_root/build_artifacts/conda-package-handling_1736345463896/work
 conda_package_streaming @ file:///home/conda/feedstock_root/build_artifacts/conda-package-streaming_1729004031731/work
@@ -121,28 +129,32 @@ gast==0.6.0
 gcs-oauth2-boto-plugin==3.0
 getdaft==0.4.3
 gitdb==4.0.11
-GitPython==3.1.40
+GitPython==3.1.44
 glfw==2.6.3
 google-api-core==2.24.2
 google-api-python-client==2.111.0
 google-apitools==0.5.32
 google-auth==2.23.4
 google-auth-httplib2==0.1.1
 google-auth-oauthlib==1.0.0
+google-cloud-core==2.4.1
+google-cloud-storage==2.14.0
+google-crc32c==1.5.0
 google-oauth==1.0.1
 google-pasta==0.2.0
 google-reauth==0.1.1
+google-resumable-media==2.6.0
 googleapis-common-protos==1.61.0
 GPy==1.13.1
 gpytorch==1.10
 graphene==3.4.3
 graphql-core==3.2.3
 graphql-relay==3.2.0
 greenlet==3.0.1
-grpcio==1.66.2
+grpcio==1.74.0
 gsutil==5.27
 gunicorn==20.1.0
-gymnasium==1.0.0
+gymnasium==1.1.1
 h11==0.16.0
 h2 @ file:///home/conda/feedstock_root/build_artifacts/h2_1733298745555/work
 h5py==3.10.0
@@ -166,6 +178,7 @@ ipykernel==6.27.1
 ipython==8.12.3
 ipython-genutils==0.2.0
 ipywidgets==8.1.3
+isodate==0.6.1
 isoduration==20.11.0
 itsdangerous==2.1.2
 jedi==0.19.1
@@ -190,14 +203,14 @@ jupyterlab_server==2.24.0
 jupyterlab_widgets==3.0.11
 keras==2.15.0
 kiwisolver==1.4.5
+kombu==5.5.4
 labmaze==1.0.6
 lazy_loader==0.4
 libclang==18.1.1
-libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1750078835/work/libmambapy
+libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1753776969/work/libmambapy
 lightgbm==4.6.0
 lightning-utilities==0.11.2
 linear-operator==0.4.0
-linkify-it-py==2.0.3
 llvmlite==0.42.0
 locket==1.0.0
 lxml==4.9.4
@@ -208,10 +221,9 @@ markdown-it-py==2.2.0
 MarkupSafe==2.1.3
 matplotlib==3.7.4
 matplotlib-inline==0.1.6
-mdit-py-plugins==0.4.2
 mdurl==0.1.2
 memray==1.10.0
-menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1750792275478/work
+menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1753546271769/work
 minigrid==2.3.1
 mistune==0.8.4
 ml-dtypes==0.3.2
@@ -240,6 +252,7 @@ netifaces==0.11.0
 networkx==3.2.1
 nevergrad==0.4.3.post7
 ninja==1.11.1.1
+nixl==0.4.0
 notebook==6.5.7
 notebook_shim==0.2.3
 numba==0.59.1
@@ -284,7 +297,6 @@ pettingzoo==1.24.3
 pexpect==4.8.0
 pickleshare==0.7.5
 pillow==10.3.0
-pip-tools==7.4.1
 platformdirs==3.11.0
 plotly==5.23.0
 pluggy==1.3.0
@@ -316,7 +328,6 @@ pynvml==11.5.0
 PyOpenGL==3.1.7
 pyOpenSSL==25.0.0
 pyparsing==3.1.1
-pyproject_hooks==1.2.0
 pyro-api==0.1.2
 pyro-ppl==1.9.1
 Pyro4==4.82
@@ -334,11 +345,10 @@ pyu2f==0.1.5
 PyYAML==6.0.1
 pyzmq==26.0.3
 qpd==0.4.4
-ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=5a7ca35580e97891618705a7a7efcc556b30cc1d3e6155605d690b18d50c4383
-redis==4.4.2
+ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=6dd296fe192d3c9953867ef02ab2645e2f5e48f49e7d4e7c1fecc7689d139cf1
 referencing==0.36.2
 regex==2024.5.15
-requests==2.32.3
+requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1733217035951/work
 requests-oauthlib==2.0.0
 requests-toolbelt==1.0.0
 responses==0.13.4
@@ -396,9 +406,8 @@ tensorflow-metadata==1.13.1
 tensorflow-probability==0.23.0
 termcolor==2.4.0
 terminado==0.18.1
-textual==4.0.0
 threadpoolctl==3.1.0
-tifffile==2024.8.30
+tifffile==2024.7.21
 timm==0.9.2
 tinycss2==1.3.0
 tinyscaler==1.2.8
@@ -423,18 +432,17 @@ transformers==4.36.2
 triad==0.9.8
 triton==2.3.0
 typeguard==2.13.3
-typer==0.16.0
+typer==0.12.3
 types-python-dateutil==2.9.0.20240316
-typing-inspection==0.4.1
 typing_extensions==4.12.2
 tzdata==2025.2
-uc-micro-py==1.0.3
 uri-template==1.3.0
 uritemplate==4.1.1
 urllib3==1.26.19
 utilsforecast==0.2.0
 uvicorn==0.22.0
 uvloop==0.21.0
+vine==5.1.0
 virtualenv==20.29.1
 wandb==0.17.0
 watchfiles==0.19.0