From cb0a40a99a63c26de6edf8fd75f99dfa22d7b2d7 Mon Sep 17 00:00:00 2001 From: zzh Date: Wed, 29 Jul 2026 23:01:56 +0800 Subject: [PATCH 01/10] feat(gpu): support vLLM 0.26 DeepSeek remote experts Signed-off-by: zzh --- afd_plugin/compat/patches/async_dp_engine.py | 83 +- .../patches/async_dp_forward_context.py | 24 +- .../compat/patches/config_validation.py | 54 +- afd_plugin/compat/patches/engine_core.py | 105 +- afd_plugin/compat/vllm.py | 2 +- afd_plugin/connectors/__init__.py | 2 + afd_plugin/connectors/gpu/p2p.py | 98 +- afd_plugin/connectors/metadata.py | 38 +- .../model_executor/models/deepseek_v2.py | 1043 ++++++------- .../npu/deepseek_v2_async_cam_forward.py | 74 +- .../models/npu/deepseek_v2_attention_gate.py | 54 +- .../v1/worker/attention_model_runner.py | 257 +++- afd_plugin/v1/worker/attention_worker.py | 24 +- afd_plugin/v1/worker/dbo.py | 12 +- afd_plugin/v1/worker/ffn_model_runner.py | 71 +- afd_plugin/v1/worker/ffn_worker.py | 35 +- afd_plugin/v1/worker/ubatch_wrapper.py | 59 +- docs/design/module/model_integration.md | 10 +- pyproject.toml | 2 +- .../compat/patches/test_async_dp_engine.py | 2 + .../patches/test_async_dp_forward_context.py | 2 + .../compat/patches/test_config_validation.py | 2 +- tests/unit/compat/patches/test_engine_core.py | 21 +- tests/unit/connectors/test_p2p_connector.py | 20 +- .../connectors/test_p2p_experts_contract.py | 196 +++ .../models/test_deepseek_v2_construction.py | 643 +++++++++ .../models/test_deepseek_v2_proxy.py | 270 ++++ .../models/test_deepseek_v2_weight_policy.py | 237 +++ .../models/test_forward_context.py | 266 +++- tests/unit/package/test_package.py | 6 +- .../v1/worker/test_attention_model_runner.py | 39 +- tests/unit/v1/worker/test_dbo.py | 35 +- tests/unit/v1/worker/test_ffn_model_runner.py | 283 +++- .../unit/v1/worker/test_runtime_classpaths.py | 119 ++ uv.lock | 1286 ++++++++++++----- 35 files changed, 4215 insertions(+), 1259 deletions(-) create mode 100644 tests/unit/connectors/test_p2p_experts_contract.py create mode 100644 tests/unit/model_executor/models/test_deepseek_v2_construction.py create mode 100644 tests/unit/model_executor/models/test_deepseek_v2_proxy.py create mode 100644 tests/unit/model_executor/models/test_deepseek_v2_weight_policy.py diff --git a/afd_plugin/compat/patches/async_dp_engine.py b/afd_plugin/compat/patches/async_dp_engine.py index 19fca2c7..0762bde4 100644 --- a/afd_plugin/compat/patches/async_dp_engine.py +++ b/afd_plugin/compat/patches/async_dp_engine.py @@ -9,7 +9,7 @@ 4. ``vllm.v1.engine.core_client.DPAsyncMPClient.add_request_async`` Why: - vLLM 0.19.1's native MoE DP path uses ``DPEngineCoreProc`` and DP wave + vLLM 0.26.0's native MoE DP path uses ``DPEngineCoreProc`` and DP wave notifications. AFD async-DP Attention ranks are connector-driven and must step independently while keeping the original DP/EP topology for expert placement and weight loading. @@ -30,7 +30,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING import vllm.v1.engine.core as engine_core_module import vllm.v1.engine.core_client as core_client_module @@ -55,13 +55,6 @@ ) from vllm.v1.executor import Executor - EngineLaunchResult: TypeAlias = tuple[ - CoreEngineProcManager | CoreEngineActorManager | None, - DPCoordinator | None, - EngineZmqAddresses, - Queue | None, - ] - # Patch reason: vLLM's MoE DP engine process uses DPEngineCoreProc, but AFD # async Attention ranks are connector-driven and must not run DP wave logic. @@ -69,10 +62,10 @@ # for AFD async Attention configs. # Signature: matches upstream; no added parameters. def run_engine_core( - *args: Any, + *args, dp_rank: int = 0, local_dp_rank: int = 0, - **kwargs: Any, + **kwargs, ): """Replace MoE DP proc selection for AFD async Attention engines.""" @@ -96,10 +89,12 @@ def run_engine_core( process_title, ) engine_core_module.decorate_logs() + if parallel_config.numa_bind: + engine_core_module.numa_utils.log_current_affinity_state(process_title) if data_parallel and vllm_config.kv_transfer_config is not None: vllm_config.kv_transfer_config.engine_id = ( - f"{vllm_config.kv_transfer_config.engine_id}_dp{local_dp_rank}" + f"{vllm_config.kv_transfer_config.engine_id}_dp{dp_rank}" ) engine_core_module.logger.debug( "Setting kv_transfer_config.engine_id to %s", @@ -124,13 +119,22 @@ def run_engine_core( parallel_config.data_parallel_rank = 0 engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs) + assert engine_core is not None + def wakeup_engine() -> None: + # Wakes up idle engine via input_queue when shutdown is requested + # Not safe in a signal handler - we may interrupt the main thread + # while it is holding the non-reentrant input_queue.mutex engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None)) signal_callback = engine_core_module.SignalCallback(wakeup_engine) - def signal_handler(signum: int, frame: object) -> None: - del signum, frame + def signal_handler(signum, frame): + signal_name = engine_core_module.signal.Signals(signum).name + engine_core_module.logger.info( + "[shutdown] EngineCore: trigger received signal=%s", + signal_name, + ) engine_core.shutdown_state = ( engine_core_module.EngineShutdownState.REQUESTED ) @@ -148,7 +152,7 @@ def signal_handler(signum: int, frame: object) -> None: engine_core.run_busy_loop() except SystemExit: - engine_core_module.logger.debug("EngineCore exiting.") + engine_core_module.logger.info_once("[shutdown] EngineCore: exiting busy loop") raise except Exception as exc: if engine_core is None: @@ -184,7 +188,14 @@ def launch_core_engines( log_stats: bool, addresses: EngineZmqAddresses, num_api_servers: int = 1, -) -> Iterator[EngineLaunchResult]: +) -> Iterator[ + tuple[ + CoreEngineProcManager | CoreEngineActorManager | None, + DPCoordinator | None, + EngineZmqAddresses, + Queue | None, + ] +]: """Disable coordinator wave mode while launching AFD async-DP engines.""" parallel_config = vllm_config.parallel_config @@ -197,7 +208,7 @@ def launch_core_engines( offline_mode = local_start_index is not None - tensor_queue = None + tensor_queue: Queue | None = None multimodal_config = vllm_config.model_config.multimodal_config if multimodal_config is not None and multimodal_config.mm_tensor_ipc == "torch_shm": tensor_queue = engine_utils_module.get_mp_context().Queue() @@ -269,10 +280,13 @@ def launch_core_engines( if parallel_config.enable_elastic_ep: handshake_local_only = False + rpc_port = ( + parallel_config.data_parallel_rpc_port or engine_utils_module.get_open_port() + ) handshake_address = engine_utils_module.get_engine_client_zmq_addr( handshake_local_only, host, - parallel_config.data_parallel_rpc_port, + rpc_port, ) if local_engines_only and dp_rank > 0: @@ -329,28 +343,6 @@ async def add_request_async( ) -> None: """Skip the DP wave ``FIRST_REQ`` notification for AFD async-DP.""" - if not is_afd_async_dp(self.vllm_config): - self._ensure_stats_update_task() - - request.current_wave = self.current_wave - request.client_index = self.client_index - - chosen_engine = self.get_core_engine_for_request(request) - to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine) - if not self.engines_running: - req_msg = core_client_module.msgspec.msgpack.encode( - ("FIRST_REQ", chosen_engine), - ) - await self.first_req_send_socket.send(req_msg) - - await to_await - - self._ensure_output_queue_task() - return None - - # ### PATCH START: AFD async-DP request wakeup - # Async-DP engines step independently, so skip the coordinator FIRST_REQ - # wakeup while preserving normal routing. self._ensure_stats_update_task() request.current_wave = self.current_wave @@ -358,10 +350,19 @@ async def add_request_async( chosen_engine = self.get_core_engine_for_request(request) to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine) + # ### PATCH START: AFD async-DP request wakeup + # Async-DP engines step independently, so skip the coordinator FIRST_REQ + # wakeup while preserving normal routing. + if not self.engines_running and not is_afd_async_dp(self.vllm_config): + req_msg = core_client_module.msgspec.msgpack.encode( + ("FIRST_REQ", chosen_engine), + ) + await self.first_req_send_socket.send(req_msg) + # ### PATCH END: AFD async-DP request wakeup + await to_await self._ensure_output_queue_task() - # ### PATCH END: AFD async-DP request wakeup def _is_afd_async_attention_config(vllm_config: VllmConfig) -> bool: diff --git a/afd_plugin/compat/patches/async_dp_forward_context.py b/afd_plugin/compat/patches/async_dp_forward_context.py index f7756991..09bc4fe8 100644 --- a/afd_plugin/compat/patches/async_dp_forward_context.py +++ b/afd_plugin/compat/patches/async_dp_forward_context.py @@ -6,7 +6,7 @@ 1. ``vllm.forward_context.set_forward_context`` Why: - vLLM 0.19.1 constructs ``DPMetadata`` and coordinates token counts across + vLLM 0.26.0 constructs ``DPMetadata`` and coordinates token counts across MoE DP ranks whenever DP size is greater than one. AFD async-DP uses the connector data flow instead of vLLM's DP metadata control plane, so those all-reduce and metadata paths must be skipped for the AFD async connector. @@ -44,8 +44,6 @@ AttentionMetadataMapping: TypeAlias = ( dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] ) - SlotMapping: TypeAlias = dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] - _FORWARD_CONTEXT_IMPORT_MODULES = ( "vllm.v1.worker.gpu_model_runner", "vllm.v1.worker.gpu.model_runner", @@ -68,8 +66,9 @@ def set_forward_context( cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, batch_descriptor: BatchDescriptor | None = None, ubatch_slices: UBatchSlices | None = None, - slot_mapping: SlotMapping | None = None, + slot_mapping: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None, skip_compiled: bool = False, + is_padding: torch.Tensor | None = None, ): """A context manager that stores the current forward context, can be attention metadata, etc. @@ -88,14 +87,20 @@ def set_forward_context( # AFD async-DP coordinates batches through connector flow, so skip vLLM's # native DPMetadata construction only for AFD async configs. if not is_afd_async_dp(vllm_config) and ( - vllm_config.parallel_config.data_parallel_size > 1 + ( + vllm_config.parallel_config.data_parallel_size > 1 + or vllm_config.parallel_config.use_sequence_parallel_moe + ) and vllm_config.parallel_config.is_moe_model is not False and (attn_metadata is not None or num_tokens is not None) ): # If num_tokens_across_dp hasn't already been initialized, then # initialize it here. Both DP padding and Microbatching will be # disabled. - if num_tokens_across_dp is None: + if ( + num_tokens_across_dp is None + and vllm_config.parallel_config.data_parallel_size > 1 + ): assert ubatch_slices is None assert num_tokens is not None _, num_tokens_across_dp, _ = ( @@ -106,6 +111,12 @@ def set_forward_context( ) ) assert num_tokens_across_dp is not None + elif num_tokens_across_dp is None: + assert num_tokens is not None + num_tokens_across_dp = forward_context_module.torch.tensor( + [num_tokens], + dtype=forward_context_module.torch.int32, + ) dp_metadata = forward_context_module.DPMetadata.make( vllm_config.parallel_config, num_tokens or 0, @@ -144,6 +155,7 @@ def set_forward_context( slot_mapping, additional_kwargs, skip_compiled, + is_padding=is_padding, ) try: diff --git a/afd_plugin/compat/patches/config_validation.py b/afd_plugin/compat/patches/config_validation.py index 17c5edb7..529d35f0 100644 --- a/afd_plugin/compat/patches/config_validation.py +++ b/afd_plugin/compat/patches/config_validation.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """Config normalization shim for AFD-owned runtime behavior. -vLLM 0.19.1 validates native microbatching by requiring a DeepEP all2all +vLLM 0.26.0 validates native microbatching by requiring a supported all2all backend. AFD ubatching uses plugin connectors instead, so this patch only relaxes that assertion for configs with active ``additional_config["afd"]``. It also replaces the platform's default worker with the role-specific AFD @@ -42,7 +42,7 @@ # validation bypass. # Signature: matches upstream; no added parameters. def create_engine_config( - self: EngineArgs, + self, usage_context: UsageContext | None = None, headless: bool = False, ) -> VllmConfig: @@ -85,7 +85,7 @@ def create_engine_config( # pipeline; keep a narrow original-function delegation so this patch only owns # AFD validation and worker normalization. # Signature: matches upstream; no added parameters. -def __post_init__(self: VllmConfig): +def __post_init__(self): """Verify configs are valid & consistent with each other.""" assert _original_vllm_config_post_init is not None @@ -144,55 +144,45 @@ def _select_afd_worker_for_auto(vllm_config: VllmConfig) -> None: def _should_relax_engine_args_backend(engine_args: EngineArgs) -> bool: if not _is_target_vllm_compatible(): return False - try: - afd_config = parse_optional_afd_config( - getattr(engine_args, "additional_config", None), - ) - except Exception: - return False + afd_config = parse_optional_afd_config(engine_args.additional_config) if afd_config is None: return False - if ( - not bool(getattr(engine_args, "enable_dbo", False)) - and int( - getattr(engine_args, "ubatch_size", 1), - ) - <= 1 - ): + if not engine_args.enable_dbo and engine_args.ubatch_size <= 1: return False - backend = getattr(engine_args, "all2all_backend", None) - return backend not in {"deepep_low_latency", "deepep_high_throughput"} + backend = engine_args.all2all_backend + return backend not in { + "deepep_low_latency", + "deepep_high_throughput", + "nixl_ep", + } def _should_relax_vllm_config_backend(vllm_config: VllmConfig) -> bool: if not _is_target_vllm_compatible(): return False - try: - afd_config = parse_optional_afd_config(vllm_config) - except Exception: - return False + afd_config = parse_optional_afd_config(vllm_config) if afd_config is None: return False - parallel_config = getattr(vllm_config, "parallel_config", None) - if parallel_config is None: - return False - if not bool(getattr(parallel_config, "use_ubatching", False)): + parallel_config = vllm_config.parallel_config + if not parallel_config.use_ubatching: return False - backend = getattr(parallel_config, "all2all_backend", None) - return backend not in {"deepep_low_latency", "deepep_high_throughput"} + backend = parallel_config.all2all_backend + return backend not in { + "deepep_low_latency", + "deepep_high_throughput", + "nixl_ep", + } def _is_target_vllm_compatible() -> bool: try: import vllm - version_value = getattr(vllm, "__version__", None) - except Exception: - version_value = None - if version_value is None: + version_value = vllm.__version__ + except (AttributeError, ImportError): return True version_text = str(version_value) if "dev" in version_text: diff --git a/afd_plugin/compat/patches/engine_core.py b/afd_plugin/compat/patches/engine_core.py index 863ffbd8..65f3cc6b 100644 --- a/afd_plugin/compat/patches/engine_core.py +++ b/afd_plugin/compat/patches/engine_core.py @@ -74,6 +74,7 @@ def __init__( # Setup Model. self.model_executor = executor_class(vllm_config) + self._pooler_config_logged = False if executor_fail_callback is not None: self.model_executor.register_failure_callback(executor_fail_callback) @@ -98,10 +99,9 @@ def __init__( ) vllm_config.scheduler_config.enable_chunked_prefill = False - scheduler_block_size = ( - vllm_config.cache_config.block_size - * vllm_config.parallel_config.decode_context_parallel_size - * vllm_config.parallel_config.prefill_context_parallel_size + scheduler_block_size, hash_block_size = core_module.resolve_kv_cache_block_sizes( + kv_cache_config, + vllm_config, ) self.scheduler = Scheduler( @@ -111,8 +111,12 @@ def __init__( include_finished_set=include_finished_set, log_stats=self.log_stats, block_size=scheduler_block_size, + hash_block_size=hash_block_size, ) self.use_spec_decode = vllm_config.speculative_config is not None + self.check_for_draft_tokens = ( + self.use_spec_decode or vllm_config.model_config.is_diffusion + ) if self.scheduler.connector is not None: # type: ignore self.model_executor.init_kv_output_aggregator(self.scheduler.connector) # type: ignore @@ -132,19 +136,19 @@ def __init__( if xfer_handshake_metadata: # xfer_handshake_metadata is list of dicts from workers - # Each dict already has structure {tp_rank: metadata} + # Each dict already has structure {(pp_rank, tp_rank): metadata} # Merge all worker dicts into a single dict - content: dict[int, Any] = {} + content: dict[tuple[int, int], Any] = {} for worker_dict in xfer_handshake_metadata: if worker_dict is not None: content.update(worker_dict) - kv_connector.set_xfer_handshake_metadata(content) + kv_connector.set_xfer_handshake_metadata_pp_aware(content) # Setup batch queue for pipeline parallelism. # Batch queue for scheduled batches. This enables us to asynchronously # schedule and execute batches, and is required by pipeline parallelism # to eliminate pipeline bubbles. - self.batch_queue_size = self.model_executor.max_concurrent_batches + self.batch_queue_size = vllm_config.max_concurrent_batches self.batch_queue = None if self.batch_queue_size > 1: core_module.logger.debug( @@ -167,7 +171,8 @@ def __init__( core_module.init_none_hash(caching_hash_fn) self.request_block_hasher = core_module.get_request_block_hasher( - scheduler_block_size, caching_hash_fn + hash_block_size, + caching_hash_fn, ) self.step_fn = self.step if self.batch_queue is None else self.step_with_batch_queue @@ -204,14 +209,21 @@ def shutdown(self): model_executor.shutdown() with suppress(Exception): gc.unfreeze() + core_module.cleanup_dist_env_and_memory() return # ### PATCH END: AFD FFN EngineCore shutdown + core_module.logger.debug_once("[shutdown] EngineCore: tearing down local resources") self.structured_output_manager.clear_backend() if self.model_executor: self.model_executor.shutdown() if self.scheduler: self.scheduler.shutdown() + gc.unfreeze() + core_module.cleanup_dist_env_and_memory() + core_module.logger.debug_once( + "[shutdown] EngineCore: local resource teardown complete" + ) # Patch reason: late-loaded AFD FFN EngineCore paths may ask for KV cache setup @@ -230,9 +242,27 @@ def _initialize_kv_caches(self, vllm_config: VllmConfig) -> KVCacheConfig: start = time.time() + core_module.register_all_kvcache_specs(vllm_config) + # Get all kv cache needed by the model kv_cache_specs = self.model_executor.get_kv_cache_specs() + if any( + getattr(spec, "non_causal", False) + for worker_specs in kv_cache_specs + for spec in worker_specs.values() + ): + if vllm_config.scheduler_config.enable_chunked_prefill: + core_module.logger.info( + "Disabling chunked prefill: model has non-causal attention layers." + ) + vllm_config.scheduler_config.enable_chunked_prefill = False + if vllm_config.cache_config.enable_prefix_caching: + core_module.logger.info( + "Disabling prefix caching: model has non-causal attention layers." + ) + vllm_config.cache_config.enable_prefix_caching = False + has_kv_cache = any(kv_cache_spec for kv_cache_spec in kv_cache_specs) if has_kv_cache: if core_module.envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: @@ -276,6 +306,12 @@ def _initialize_kv_caches(self, vllm_config: VllmConfig) -> KVCacheConfig: vllm_config.cache_config.block_size = min( g.kv_cache_spec.block_size for g in kv_cache_groups ) + num_tokens, max_concurrency = core_module.get_kv_cache_capacity( + vllm_config, + scheduler_kv_cache_config, + ) + vllm_config.cache_config.kv_cache_size_tokens = num_tokens + vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency vllm_config.validate_block_size() @@ -283,11 +319,30 @@ def _initialize_kv_caches(self, vllm_config: VllmConfig) -> KVCacheConfig: self.model_executor.initialize_from_config(kv_cache_configs) elapsed = time.time() - start - core_module.logger.info_once( - "init engine (profile, create kv cache, warmup model) took %.2f seconds", - elapsed, - scope="local", - ) + compile_time = vllm_config.compilation_config.compilation_time + encoder_compile_time = vllm_config.compilation_config.encoder_compilation_time + if encoder_compile_time > 0: + core_module.logger.info_once( + "init engine (profile, create kv cache, warmup model) took " + "%.2f s (compilation: %.2f s — language_model: %.2f s, " + "encoder: %.2f s)", + elapsed, + compile_time + encoder_compile_time, + compile_time, + encoder_compile_time, + ) + elif compile_time > 0: + core_module.logger.info_once( + "init engine (profile, create kv cache, warmup model) took " + "%.2f s (compilation: %.2f s)", + elapsed, + compile_time, + ) + else: + core_module.logger.info_once( + "init engine (profile, create kv cache, warmup model) took %.2f s", + elapsed, + ) return scheduler_kv_cache_config @@ -306,10 +361,14 @@ def run_busy_loop(self): # ### PATCH END: AFD FFN connector busy loop if isinstance(self, core_module.DPEngineCoreProc): + """Core busy loop of the EngineCore for data parallel case.""" + # Loop until process is sent a SIGINT or SIGTERM while self._handle_shutdown(): # 1) Poll the input queue until there is work to do. self._process_input_queue() + # Publish request counts before and after GPU step to ensure freshness. + self._maybe_publish_request_counts() if self.eep_scaling_state is not None: _ = self.eep_scaling_state.progress() @@ -328,9 +387,21 @@ def run_busy_loop(self): # All engines are idle. continue - # We are in a running state and so must execute a dummy pass - # if the model didn't execute any ready requests. - self.execute_dummy_batch() + # Execute a dummy pass when no ready requests ran, unless the + # engine is sleeping. + elif not self.model_executor.is_sleeping: + with self.capture_iteration_details(None) as iteration_details: + self.execute_dummy_batch() + if iteration_details is not None and not self.has_coordinator: + stats = self._make_iteration_details_stats(iteration_details) + self.output_queue.put_nowait( + ( + 0, + core_module.EngineCoreOutputs( + scheduler_stats=stats, + ), + ) + ) # 3) All-reduce operation to determine global unfinished reqs. self.engines_running = self._has_global_unfinished_reqs( diff --git a/afd_plugin/compat/vllm.py b/afd_plugin/compat/vllm.py index e1786a0b..dd1915f7 100644 --- a/afd_plugin/compat/vllm.py +++ b/afd_plugin/compat/vllm.py @@ -9,7 +9,7 @@ from importlib.metadata import PackageNotFoundError, version from typing import Final -TARGET_VLLM_VERSION: Final[str] = "0.19.1" +TARGET_VLLM_VERSION: Final[str] = "0.26.0" def _parse_release(value: str) -> tuple[int, int, int]: diff --git a/afd_plugin/connectors/__init__.py b/afd_plugin/connectors/__init__.py index a8de61ab..ed331c74 100644 --- a/afd_plugin/connectors/__init__.py +++ b/afd_plugin/connectors/__init__.py @@ -12,6 +12,7 @@ AFDA2FTransferPayload, AFDControlPayload, AFDDPMetadata, + AFDExpertRoutingSpec, AFDF2ATransferPayload, AFDForwardContextMetadata, AFDSingleDPMetadata, @@ -24,6 +25,7 @@ "AFDConnectorBase", "AFDControlPlane", "ConnectorExtraInfo", + "AFDExpertRoutingSpec", "AFDTransferState", "AFDTransferContext", "AFDConnectorFactory", diff --git a/afd_plugin/connectors/gpu/p2p.py b/afd_plugin/connectors/gpu/p2p.py index 712ad4d1..b7c04bd3 100644 --- a/afd_plugin/connectors/gpu/p2p.py +++ b/afd_plugin/connectors/gpu/p2p.py @@ -82,6 +82,7 @@ AFDA2FTransferPayload, AFDControlPayload, AFDDPMetadata, + AFDExpertRoutingSpec, AFDTransferContext, AFDTransferMetadata, recv_control_payload, @@ -207,6 +208,7 @@ def __init__( self.e2a_pynccl: PyNcclCommunicator | None = None self.a2e_comm_id: int | None = None self.e2a_comm_id: int | None = None + self._p2p_ordering_token: torch.Tensor | None = None self.control_plane = P2pNcclAFDControlPlane(self) def close(self) -> None: @@ -228,6 +230,7 @@ def close(self) -> None: if callable(shutdown): shutdown() setattr(self, communicator_name, None) + self._p2p_ordering_token = None self._initialized = False def init_afd_connector(self) -> None: @@ -282,6 +285,11 @@ def init_afd_connector(self) -> None: device=self.local_rank, ) self.e2a_comm_id = _register_comm(self.e2a_pynccl) + self._p2p_ordering_token = torch.zeros( + 1, + dtype=torch.int64, + device=torch.device("cuda", self.local_rank), + ) if self.mapping.participates_in_dp_metadata_group: self.p2p_pg = init_afd_process_group( @@ -313,7 +321,7 @@ def send_attn_output( whose leading dimension matches ``context.metadata.total_tokens``. context: Per-transfer context describing the token layout. - **kwargs: Unused; accepted for interface compatibility. + **kwargs: Optional ``router_logits`` tensor sent after hidden states. Raises: ValueError: If the tensor shape does not match the metadata token @@ -329,12 +337,27 @@ def send_attn_output( f"hidden_states shape {hidden_states.shape!r} does not match " f"AFD metadata token count {metadata.total_tokens}", ) + router_logits: torch.Tensor | None = kwargs.get("router_logits") + if ( + router_logits is not None + and router_logits.shape[0] != hidden_states.shape[0] + ): + raise ValueError( + "router_logits and hidden_states must have equal token counts", + ) self._send_hidden_states( hidden_states, 0, self.a2e_group, self.a2e_comm_id, ) + if router_logits is not None: + self._send_hidden_states( + router_logits, + 0, + self.a2e_group, + self.a2e_comm_id, + ) def recv_ffn_output( self, @@ -388,7 +411,7 @@ def recv_attn_output( Args: ubatch_idx: Stage/microbatch index to receive. Defaults to ``0``. - **kwargs: Unused; accepted for interface compatibility. + **kwargs: Optional ``routing_spec`` for an experts-boundary layer. Returns: ``AFDA2FTransferPayload`` with the concatenated hidden states and a @@ -399,7 +422,9 @@ def recv_attn_output( RuntimeError: If the connector is not initialized or the subgroup has no Attention peers. """ + routing_spec: AFDExpertRoutingSpec | None = kwargs.get("routing_spec") hidden_states_list: list[torch.Tensor] = [] + router_logits_list: list[torch.Tensor] = [] for src in range(1, self.group_size): tensor_metadata = self._recv_attn_tensor_metadata_list.get( @@ -420,6 +445,25 @@ def recv_attn_output( ref_tensor=ref_tensor, ), ) + if routing_spec is not None: + router_metadata = _TensorMetadata( + device=tensor_metadata.device, + dtype=routing_spec.router_logits_dtype, + size=torch.Size( + [ + tensor_metadata.size[0], + routing_spec.router_logits_width, + ], + ), + ) + router_logits_list.append( + self._recv_hidden_states( + src, + self.a2e_group, + self.a2e_comm_id, + router_metadata, + ), + ) if not hidden_states_list: raise RuntimeError("P2P FFN rank has no Attention peers") @@ -428,6 +472,14 @@ def recv_attn_output( if len(hidden_states_list) > 1 else hidden_states_list[0] ) + router_logits = None + if routing_spec is not None: + router_logits = ( + torch.cat(router_logits_list, dim=0) + if len(router_logits_list) > 1 + else router_logits_list[0] + ) + metadata = AFDTransferMetadata.create_ffn_metadata( layer_idx=0, stage_idx=ubatch_idx, @@ -435,7 +487,10 @@ def recv_attn_output( ) return AFDA2FTransferPayload( hidden_states=hidden_states, - context=AFDTransferContext(metadata=metadata), + context=AFDTransferContext( + metadata=metadata, + ), + router_logits=router_logits, ) def send_ffn_output( @@ -525,9 +580,12 @@ def _send_hidden_states( raise ValueError(f"invalid P2P destination rank {dst}") if getattr(hidden_states, "is_cpu", False): raise ValueError("P2P hidden states must be on GPU") + if self._p2p_ordering_token is None: + raise RuntimeError("P2P connector ordering token is not initialized") torch.ops.vllm.afd_p2p_send( hidden_states, + self._p2p_ordering_token, dst, comm_id, ) @@ -576,7 +634,14 @@ def _recv_hidden_states( dtype=tensor_metadata.dtype, device=tensor_metadata.device, ) - torch.ops.vllm.afd_p2p_recv(hidden_states, src, comm_id) + if self._p2p_ordering_token is None: + raise RuntimeError("P2P connector ordering token is not initialized") + torch.ops.vllm.afd_p2p_recv( + hidden_states, + self._p2p_ordering_token, + src, + comm_id, + ) return hidden_states @@ -792,7 +857,7 @@ def _register_comm(communicator: PyNcclCommunicator) -> int: def _register_p2p_custom_ops() -> None: - """Register ``afd_p2p_send`` / ``afd_p2p_recv`` as vLLM custom ops. + """Register the ordered AFD P2P send and receive custom ops. Wrapping ``PyNcclCommunicator.send()`` / ``recv()`` in custom ops with fake implementations keeps the transfers traceable by ``torch.compile`` @@ -806,12 +871,14 @@ def _register_p2p_custom_ops() -> None: def afd_p2p_send_impl( tensor: torch.Tensor, + ordering_token: torch.Tensor, dst: int, comm_id: int, ) -> None: communicator = _AFD_COMMUNICATORS.get(comm_id) if communicator is None: raise RuntimeError(f"AFD communicator id {comm_id} is not registered") + ordering_token.add_(1) communicator.send( tensor, dst, @@ -821,22 +888,34 @@ def afd_p2p_send_impl( def afd_p2p_send_fake( tensor: torch.Tensor, + ordering_token: torch.Tensor, dst: int, comm_id: int, ) -> None: pass - def afd_p2p_recv_impl(out: torch.Tensor, src: int, comm_id: int) -> None: + def afd_p2p_recv_impl( + out: torch.Tensor, + ordering_token: torch.Tensor, + src: int, + comm_id: int, + ) -> None: communicator = _AFD_COMMUNICATORS.get(comm_id) if communicator is None: raise RuntimeError(f"AFD communicator id {comm_id} is not registered") + ordering_token.add_(1) communicator.recv( out, src, stream=torch.cuda.current_stream(out.device), ) - def afd_p2p_recv_fake(out: torch.Tensor, src: int, comm_id: int) -> None: + def afd_p2p_recv_fake( + out: torch.Tensor, + ordering_token: torch.Tensor, + src: int, + comm_id: int, + ) -> None: pass def register_one( @@ -861,16 +940,15 @@ def register_one( register_one( op_name="afd_p2p_send", op_func=afd_p2p_send_impl, - mutates_args=["tensor"], + mutates_args=["ordering_token"], fake_impl=afd_p2p_send_fake, ) register_one( op_name="afd_p2p_recv", op_func=afd_p2p_recv_impl, - mutates_args=["out"], + mutates_args=["out", "ordering_token"], fake_impl=afd_p2p_recv_fake, ) - _AFD_CUSTOM_OPS_REGISTERED = True diff --git a/afd_plugin/connectors/metadata.py b/afd_plugin/connectors/metadata.py index 7267b565..64944a06 100644 --- a/afd_plugin/connectors/metadata.py +++ b/afd_plugin/connectors/metadata.py @@ -9,7 +9,7 @@ from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import torch from vllm.forward_context import DPMetadata @@ -145,17 +145,20 @@ def __post_init__(self) -> None: class AFDTransferState: """Base class for backend-specific connector transfer state. - Backends subclass ``AFDTransferState`` to carry whatever per-transfer state - they read back themselves between the receive and send phases of a single - AFD Attention/FFN exchange: routed/shared MoE compute payloads, handles, - HCCL endpoint names, active-token masks, sizes, receive-side token counts, - and so on. The concrete subclass instance is held directly by - ``AFDTransferContext.states``. The hidden-state tensor is kept separately on - ``AFDA2FTransferPayload``. Backends that route no per-transfer payload (the - GPU P2P connector) leave ``AFDTransferContext.states`` as ``None``. + Backends subclass ``AFDTransferState`` to carry per-transfer state between + the receive and send phases of one AFD exchange. The concrete instance is + held by ``AFDTransferContext.states``. Transfers without additional state + leave the slot as ``None``. """ +class AFDExpertRoutingSpec(NamedTuple): + """Model-owned contract for receiving optional experts routing tensors.""" + + router_logits_width: int + router_logits_dtype: torch.dtype + + @dataclass(slots=True) class AFDTransferMetadata: """Communication metadata for one AFD Attention/FFN exchange. @@ -228,13 +231,9 @@ class AFDTransferContext: layer/stage/token layout to an optional ``AFDTransferState`` subclass carrying the backend's per-transfer payloads. - ``states`` is a pluggable slot: backends that route no per-transfer payload - through the context (the GPU P2P connector) leave it ``None``, while - backends that do (CAMP2P, async CAM) attach their own ``AFDTransferState`` - subclass (``CAMP2PTransferState`` / ``AFDAsyncTransferState``) from the - connector method that produces the payload. Consumers that read ``states`` - therefore only do so on the paths of the backend that populated it. Keeping - the default ``None`` — rather than a + ``states`` is a pluggable slot for backend-specific connectors. Ordinary + transfers leave it ``None``. Consumers read it only on paths that populate + it. Keeping the default ``None`` — rather than a ``field(default_factory=AFDTransferState)`` — also keeps the generated ``__init__`` traceable by ``torch.compile``/Dynamo, which fails on the factory call and is exercised where attention forwards build the context @@ -242,8 +241,8 @@ class AFDTransferContext: Attributes: metadata: Metadata describing the layer/stage and token layout. - states: Optional backend-specific ``AFDTransferState`` subclass. ``None`` - for backends that route no payload through the context. + states: Optional ``AFDTransferState`` subclass. ``None`` when the + transfer has no additional state. """ metadata: AFDTransferMetadata @@ -263,10 +262,12 @@ class AFDA2FTransferPayload: hidden_states: Hidden-state tensor received from the Attention side. context: Transfer context describing the received transfer, including transfer metadata and backend-produced transfer state. + router_logits: Optional Attention-side routing tensor. """ hidden_states: torch.Tensor context: AFDTransferContext + router_logits: torch.Tensor | None = None @dataclass(slots=True) @@ -403,6 +404,7 @@ def recv_control_payload( __all__ = [ + "AFDExpertRoutingSpec", "AFDTransferState", "AFDTransferContext", "AFDTransferMetadata", diff --git a/afd_plugin/model_executor/models/deepseek_v2.py b/afd_plugin/model_executor/models/deepseek_v2.py index e7f98b7c..add1a88a 100644 --- a/afd_plugin/model_executor/models/deepseek_v2.py +++ b/afd_plugin/model_executor/models/deepseek_v2.py @@ -8,52 +8,36 @@ hidden states between the Attention and FFN roles through the AFD connector. """ -import typing -from collections.abc import Callable, Iterable -from itertools import islice -from typing import Any +from collections.abc import Iterable, Iterator +from typing import Any, TypeAlias import torch import torch.nn as nn -from vllm.config import VllmConfig, get_current_vllm_config +from transformers import DeepseekV2Config, DeepseekV3Config, GlmMoeDsaConfig +from vllm.config import ParallelConfig, VllmConfig from vllm.forward_context import get_forward_context from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe.shared_fused_moe import ( - SharedFusedMoE, -) from vllm.model_executor.layers.linear import ReplicatedLinear -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models import deepseek_v2 as native -from vllm.model_executor.models.deepseek_v2 import ( - get_spec_layer_idx_from_weight_name, -) -from vllm.model_executor.models.utils import is_pp_missing_parameter -try: - from vllm_ascend.ascend_config import get_ascend_config -except ImportError: - get_ascend_config = None - -from afd_plugin.config import parse_optional_afd_config +from afd_plugin.config import AFD_ASYNC_CONNECTOR, parse_afd_config from afd_plugin.connectors import ( + AFDExpertRoutingSpec, AFDF2ATransferPayload, - AFDForwardContextMetadata, AFDTransferContext, AFDTransferMetadata, ) -from afd_plugin.model_executor.models import ( - get_afd_metadata_from_forward_context, - get_async_moe_ubatch_metadata_from_forward_context, -) +from afd_plugin.model_executor.models import get_afd_metadata_from_forward_context from afd_plugin.v1.worker.dbo import maybe_apply_dbo_yield logger = init_logger(__name__) +_DeepseekAdapterConfig: TypeAlias = ( + DeepseekV2Config | DeepseekV3Config | GlmMoeDsaConfig +) + -def _is_moe_layer(config: object, layer_idx: int) -> bool: +def _is_moe_layer(config: _DeepseekAdapterConfig, layer_idx: int) -> bool: moe_layer_freq = getattr(config, "moe_layer_freq", 1) return ( config.n_routed_experts is not None @@ -62,22 +46,270 @@ def _is_moe_layer(config: object, layer_idx: int) -> bool: ) +_ATTENTION_ROLE = frozenset(("attention",)) +_FFN_ROLE = frozenset(("ffn",)) +_BOTH_ROLES = frozenset(("attention", "ffn")) + + +def _weight_layer_path(name: str) -> tuple[int, str, tuple[str, ...]] | None: + """Return ``(layer index, stage, remainder)`` for a decoder weight.""" + parts = name.split(".") + for marker_idx, part in enumerate(parts[:-2]): + if part != "layers": + continue + try: + layer_idx = int(parts[marker_idx + 1]) + except ValueError: + continue + return layer_idx, parts[marker_idx + 2], tuple(parts[marker_idx + 3 :]) + return None + + +def _checkpoint_weight_roles( + name: str, + config: _DeepseekAdapterConfig, + *, + compute_gate_on_attention: bool, +) -> frozenset[str]: + """Classify one native checkpoint path by its AFD execution owner.""" + layer_path = _weight_layer_path(name) + if layer_path is None: + return _BOTH_ROLES + + layer_idx, stage, remainder = layer_path + if stage == "self_attn": + return _ATTENTION_ROLE + if stage != "mlp": + return _BOTH_ROLES + + if not _is_moe_layer(config, layer_idx): + return _ATTENTION_ROLE if compute_gate_on_attention else _FFN_ROLE + + is_moe_gate = bool(remainder) and remainder[0] == "gate" + if is_moe_gate and compute_gate_on_attention: + return _BOTH_ROLES + return _FFN_ROLE + + +def _iter_role_weights( + weights: Iterable[tuple[str, torch.Tensor]], + *, + role: str, + config: _DeepseekAdapterConfig, + compute_gate_on_attention: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + """Consume a checkpoint iterator once and retain this role's paths.""" + for name, loaded_weight in weights: + if role in _checkpoint_weight_roles( + name, + config, + compute_gate_on_attention=compute_gate_on_attention, + ): + yield name, loaded_weight + + +class RemoteFFNProxy(nn.Module): + """Parameter-free FFN stage executed through the AFD connector.""" + + def __init__(self, *, layer_idx: int) -> None: + super().__init__() + self.layer_idx = layer_idx + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self._send_and_receive(hidden_states) + + def _send_and_receive( + self, + hidden_states: torch.Tensor, + **send_kwargs: torch.Tensor, + ) -> torch.Tensor: + afd_metadata = get_afd_metadata_from_forward_context() + if afd_metadata is None: + raise RuntimeError("RemoteFFNProxy requires AFD forward metadata") + forward_context = get_forward_context() + stage_idx = int( + getattr(forward_context, "ubatch_idx", afd_metadata.stage_idx), + ) + afd_metadata.stage_idx = stage_idx + metadata = AFDTransferMetadata.create_attention_metadata( + layer_idx=self.layer_idx, + stage_idx=stage_idx, + seq_len=int(hidden_states.shape[0]), + ) + context = AFDTransferContext(metadata=metadata) + afd_metadata.connector.send_attn_output( + hidden_states, + context, + **send_kwargs, + ) + hidden_states = maybe_apply_dbo_yield( + hidden_states, + role="attention", + ) + return afd_metadata.connector.recv_ffn_output( + ref_tensor=hidden_states, + ubatch_idx=stage_idx, + ) + + +class AFDAttentionFusedMoE(RemoteFFNProxy): + """Parameter-free native-MoE experts proxy for the Attention runtime.""" + + def __init__( + self, + *, + layer_idx: int, + is_internal_router: bool, + ) -> None: + super().__init__(layer_idx=layer_idx) + self.is_internal_router = is_internal_router + + def forward( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + input_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + if input_ids is not None: + raise NotImplementedError( + "experts-boundary input_ids transport is not implemented", + ) + send_kwargs = ( + {} if self.is_internal_router else {"router_logits": router_logits} + ) + return self._send_and_receive(hidden_states, **send_kwargs) + + def update_expert_map(self) -> None: + """Satisfy the native EPLB model interface without local experts.""" + + +class GateOnlyRemoteMoE(RemoteFFNProxy): + """Attention-side MoE gate with experts delegated to the FFN role.""" + + def __init__( + self, + *, + config: _DeepseekAdapterConfig, + layer_idx: int, + prefix: str, + vllm_config: VllmConfig, + ) -> None: + super().__init__(layer_idx=layer_idx) + self.vllm_config = vllm_config + self.config = config + self.top_k = int(config.num_experts_per_tok) + self.gate = ReplicatedLinear( + config.hidden_size, + config.n_routed_experts, + bias=False, + quant_config=None, + prefix=f"{prefix}.gate", + ) + if getattr(config, "topk_method", None) == "noaux_tc": + self.gate.e_score_correction_bias = nn.Parameter( + torch.empty(config.n_routed_experts, dtype=torch.float32), + ) + else: + self.gate.e_score_correction_bias = None + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + from afd_plugin.model_executor.models.npu import ( + deepseek_v2_attention_gate, + ) + + topk_weights, topk_ids, router_logits = ( + deepseek_v2_attention_gate.compute_gate_topk( + gate=self.gate, + vllm_config=self.vllm_config, + config=self.config, + top_k=self.top_k, + hidden_states=hidden_states, + ) + ) + return self._send_and_receive( + hidden_states, + topk_weights=topk_weights, + topk_ids=topk_ids, + router_logits=router_logits, + ) + + +class AFDDeepseekV2RemoteExpertsMoE(native.DeepseekV2MoE): + """Native DeepSeek MoE forward with parameter-free remote experts.""" + + # Patch reason: native DeepseekV2MoE constructs local routed/shared experts. + # Patch functionality: preserve the native MoE forward contract while + # constructing only the gate owned by Attention and a parameter-free proxy. + # Signature: AFD-owned; adds layer_idx and gate_placement and omits + # quant_config because no local expert kernel is constructed. + # Upstream: vLLM v0.26.0, vllm/model_executor/models/deepseek_v2.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 + def __init__( + self, + *, + config: _DeepseekAdapterConfig, + parallel_config: ParallelConfig, + layer_idx: int, + prefix: str, + compute_gate_on_attention: bool, + ) -> None: + # ### PATCH START: construct a remote-experts native MoE shell. + nn.Module.__init__(self) + self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe + + router_dtype = native._get_moe_router_dtype(config) + if compute_gate_on_attention: + self.gate = native.GateLinear( + config.hidden_size, + config.n_routed_experts, + out_dtype=router_dtype, + prefix=f"{prefix}.gate", + ) + if getattr(config, "topk_method", None) == "noaux_tc": + self.gate.e_score_correction_bias = nn.Parameter( + torch.empty(config.n_routed_experts, dtype=torch.float32), + ) + else: + self.gate.e_score_correction_bias = None + else: + self.gate = None + + ep_size = native.get_ep_group().device_group.size() + self.n_routed_experts = int(config.n_routed_experts) + self.n_shared_experts = int(config.n_shared_experts) + self.n_redundant_experts = parallel_config.eplb_config.num_redundant_experts + self.n_logical_experts = self.n_routed_experts + self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts + self.n_local_physical_experts = self.n_physical_experts // ep_size + self.experts = AFDAttentionFusedMoE( + layer_idx=layer_idx, + is_internal_router=not compute_gate_on_attention, + ) + # ### PATCH END: construct a remote-experts native MoE shell. + + class AFDDeepseekV2DecoderLayer(native.DeepseekV2DecoderLayer): """DeepSeek decoder layer with separable Attention and FFN execution.""" - def __init__(self, *args: Any, **kwargs: Any) -> None: - vllm_config = args[0] if args else kwargs.get("vllm_config") - afd_config = parse_optional_afd_config(vllm_config, validate=False) - afd_role = afd_config.role if afd_config is not None else None - - if afd_role is None: - super().__init__(*args, **kwargs) - self.afd_role = None - return + # Patch reason: native DeepSeek constructs both Attention and FFN modules. + # Patch functionality: construct only the modules owned by the active AFD role. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/model_executor/models/deepseek_v2.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 + def __init__( + self, + vllm_config: VllmConfig, + prefix: str, + config: DeepseekV2Config | None = None, + topk_indices_buffer: torch.Tensor | None = None, + ) -> None: + # ### PATCH START: require an explicit AFD role before allocation. + afd_config = parse_afd_config(vllm_config, validate=False) + afd_role = afd_config.role torch.nn.Module.__init__(self) + # ### PATCH END - config = args[2] if len(args) > 2 else kwargs.get("config") if config is None: config = vllm_config.model_config.hf_config model_config = vllm_config.model_config @@ -85,24 +317,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: quant_config = vllm_config.quant_config parallel_config = vllm_config.parallel_config - self.vllm_config = vllm_config - self.config = config - self.afd_config = afd_config self.hidden_size = config.hidden_size max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - prefix = args[1] if len(args) > 1 else kwargs.get("prefix", "") + moe_layer_freq = getattr(config, "moe_layer_freq", 1) + # DecoderLayers are created with `make_layers` which passes the prefix + # with the layer's index. layer_idx = int(prefix.split(sep=".")[-1]) self.layer_idx = layer_idx - self.is_moe_layer = _is_moe_layer(config, layer_idx) - self.compute_gate_on_attention = bool(afd_config.compute_gate_on_attention) - if ( - self.compute_gate_on_attention - and native.current_platform.device_type != "npu" - ): - raise RuntimeError( - "DeepSeekV2 compute_gate_on_attention is supported only on NPU", - ) - self.top_k = int(config.num_experts_per_tok) qk_nope_head_dim = getattr(config, "qk_nope_head_dim", 0) qk_rope_head_dim = getattr(config, "qk_rope_head_dim", 0) @@ -111,11 +332,45 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: use_mha = config.model_type == "deepseek" or all( dim == 0 for dim in (qk_nope_head_dim, qk_rope_head_dim) ) + self.use_mha = use_mha - self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) - self.afd_role = afd_role - # Create only the modules needed for this role. + is_moe_layer = ( + config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % moe_layer_freq == 0 + ) + # TODO(wentao): enable SP MoE with PP after the PP boundary logic can safely + # send/receive sequence-parallel hidden_states across stages. + self.use_sequence_parallel_moe = ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + and is_moe_layer + ) + + # ### PATCH START: construct only the stage owned by this AFD role. + self.vllm_config = vllm_config + self.config = config + self.is_moe_layer = is_moe_layer + self.compute_gate_on_attention = bool( + afd_config.compute_gate_on_attention, + ) + device_type = native.current_platform.device_type + self.uses_remote_experts = device_type == "cuda" and self.is_moe_layer + if ( + afd_role == "attention" + and self.uses_remote_experts + and parallel_config.enable_eplb + ): + raise RuntimeError( + "CUDA remote experts do not support EPLB on the Attention role", + ) + if self.compute_gate_on_attention and device_type not in ("cuda", "npu"): + raise RuntimeError( + "DeepSeekV2 compute_gate_on_attention requires CUDA or NPU", + ) + self.top_k = int(config.num_experts_per_tok) + if afd_role == "attention": attn_cls = ( native.DeepseekAttention @@ -140,26 +395,26 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", - topk_indices_buffer=kwargs.get("topk_indices_buffer"), + topk_indices_buffer=topk_indices_buffer, + reduce_results=not self.use_sequence_parallel_moe, ) - # NPU-only: non-NPU platforms are rejected before this branch. - if self.compute_gate_on_attention and self.is_moe_layer: - self.gate = ReplicatedLinear( - config.hidden_size, - config.n_routed_experts, - bias=False, - quant_config=None, - prefix=f"{prefix}.gate", + if self.uses_remote_experts: + self.mlp = AFDDeepseekV2RemoteExpertsMoE( + config=config, + parallel_config=parallel_config, + layer_idx=layer_idx, + prefix=f"{prefix}.mlp", + compute_gate_on_attention=self.compute_gate_on_attention, ) - if getattr(config, "topk_method", None) == "noaux_tc": - self.gate.e_score_correction_bias = nn.Parameter( - torch.empty(config.n_routed_experts, dtype=torch.float32) - ) - else: - self.gate.e_score_correction_bias = None - - if self.compute_gate_on_attention and not self.is_moe_layer: + elif self.compute_gate_on_attention and self.is_moe_layer: + self.mlp = GateOnlyRemoteMoE( + config=config, + layer_idx=layer_idx, + prefix=f"{prefix}.mlp", + vllm_config=vllm_config, + ) + elif self.compute_gate_on_attention: self.mlp = native.DeepseekV2MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, @@ -167,17 +422,29 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: quant_config=quant_config, prefix=f"{prefix}.mlp", ) + else: + self.mlp = RemoteFFNProxy(layer_idx=layer_idx) elif afd_role == "ffn": + self.self_attn = native.PPMissingLayer() if self.compute_gate_on_attention and not self.is_moe_layer: - pass + self.mlp = native.PPMissingLayer() elif self.is_moe_layer: self.mlp = native.DeepseekV2MoE( config=config, parallel_config=parallel_config, quant_config=quant_config, prefix=f"{prefix}.mlp", + # aiter applies routed_scaling_factor internally + apply_routed_scale_to_output=( + not native.rocm_aiter_ops.is_fused_moe_enabled() + ), ) + if self.compute_gate_on_attention and device_type == "cuda": + # Keep the native gate parameter and path for loader/model + # compatibility, but configure the runner to consume the + # router logits transferred from Attention. + self.mlp.experts.gate = None else: self.mlp = native.DeepseekV2MLP( hidden_size=config.hidden_size, @@ -186,6 +453,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: quant_config=quant_config, prefix=f"{prefix}.mlp", ) + # ### PATCH END self.input_layernorm = native.RMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -193,52 +461,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.post_attention_layernorm = native.RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - llama_4_scaling: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - if residual is None: - residual = hidden_states.clone() - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - - attn_kwargs: dict[str, torch.Tensor | None] = { - "positions": positions, - "hidden_states": hidden_states, - } - if not self.use_mha: - attn_kwargs["llama_4_scaling"] = llama_4_scaling - hidden_states = self.self_attn(**attn_kwargs) - - if ( - not isinstance(self.self_attn, native.DeepseekAttention) - and hidden_states.dtype == torch.float16 - ): - hidden_states *= 1.0 / self.routed_scaling_factor - if self.layer_idx == 0: - residual *= 1.0 / self.routed_scaling_factor - - hidden_states, residual = self.post_attention_layernorm( - hidden_states, - residual, - ) - if self.afd_role == "attention" and not ( - self.compute_gate_on_attention and not self.is_moe_layer - ): - return hidden_states, residual - - hidden_states = self.mlp(hidden_states) - if ( - isinstance(self.mlp, native.DeepseekV2MLP) - and hidden_states.dtype == torch.float16 - ): - hidden_states *= 1.0 / self.routed_scaling_factor - return hidden_states, residual + self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) def compute_attn_output( self, @@ -313,7 +536,10 @@ def compute_ffn_output( "Dense DeepSeek layers are computed on the Attention side " "when compute_gate_on_attention=true", ) - if self.compute_gate_on_attention: + if ( + self.compute_gate_on_attention + and native.current_platform.device_type == "npu" + ): if group_list is None: raise RuntimeError( "compute_gate_on_attention FFN MoE compute requires group_list", @@ -334,6 +560,10 @@ def compute_ffn_output( group_list_type=group_list_type, ) return output + if self.compute_gate_on_attention: + raise RuntimeError( + "GPU Attention-side gate must call compute_experts_output", + ) hidden_states = self.mlp(hidden_states) if ( isinstance(self.mlp, native.DeepseekV2MLP) @@ -342,28 +572,65 @@ def compute_ffn_output( hidden_states *= 1.0 / self.routed_scaling_factor return hidden_states + def compute_experts_output( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + ) -> torch.Tensor: + """Execute the native external-router runner on the FFN role.""" + if not self.compute_gate_on_attention or not self.is_moe_layer: + raise RuntimeError( + "compute_experts_output requires an Attention-side-gate MoE layer", + ) + if not isinstance(self.mlp, native.DeepseekV2MoE): + raise RuntimeError("FFN role does not own a native DeepSeek MoE") + if self.mlp.experts.is_internal_router: + raise RuntimeError("FFN native runner must use external routing") + return self.mlp.experts( + hidden_states=hidden_states, + router_logits=router_logits, + ) + @native.support_torch_compile -class AFDDeepseekV2Model(torch.nn.Module): +class AFDDeepseekV2Model(native.DeepseekV2Model): """DeepSeek model wrapper that routes Attention outputs through AFD.""" fall_back_to_pt_during_load = False - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - + # Patch reason: native DeepSeek always creates native Decoder layers. + # Patch functionality: create role-aware AFD layers without full allocation. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/model_executor/models/deepseek_v2.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + # ### PATCH START: require AFD activation and avoid native allocation. + afd_config = parse_afd_config(vllm_config, validate=False) + if bool( + getattr( + vllm_config.parallel_config, + "use_sequence_parallel_moe", + False, + ), + ): + raise RuntimeError( + "AFD DeepSeek does not support sequence-parallel MoE", + ) + torch.nn.Module.__init__(self) self.vllm_config = vllm_config self.compilation_config = vllm_config.compilation_config + self.afd_config = afd_config + # ### PATCH END config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - self.afd_config = parse_optional_afd_config(vllm_config, validate=False) self.config = config self.device = native.current_platform.device_type - + self.hidden_size = config.hidden_size self.vocab_size = config.vocab_size self.is_v32 = hasattr(config, "index_topk") - if self.is_v32: + # ### PATCH START: allocate the Indexer buffer only on Attention. + if self.is_v32 and afd_config.role == "attention": topk_tokens = config.index_topk topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, @@ -373,39 +640,52 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: ) else: topk_indices_buffer = None + # ### PATCH END if native.get_pp_group().is_first_rank: self.embed_tokens = native.VocabParallelEmbedding( config.vocab_size, - config.hidden_size, + self.hidden_size, quant_config=quant_config, prefix=f"{prefix}.embed_tokens", ) else: self.embed_tokens = native.PPMissingLayer() + # ### PATCH START: use the pinned role-aware DecoderLayer constructor. self.start_layer, self.end_layer, self.layers = native.make_layers( config.num_hidden_layers, lambda prefix: AFDDeepseekV2DecoderLayer( - vllm_config, - prefix, + vllm_config=vllm_config, + prefix=prefix, topk_indices_buffer=topk_indices_buffer, ), prefix=f"{prefix}.layers", ) + # ### PATCH END if native.get_pp_group().is_last_rank: - self.norm = native.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm = native.RMSNorm(self.hidden_size, eps=config.rms_norm_eps) else: self.norm = native.PPMissingLayer() self.make_empty_intermediate_tensors = ( native.make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], - config.hidden_size, + self.hidden_size, ) ) self.aux_hidden_state_layers = tuple[int, ...]() + # Needed by load_weights + qk_nope_head_dim = getattr(config, "qk_nope_head_dim", 0) + qk_rope_head_dim = getattr(config, "qk_rope_head_dim", 0) + self.use_mha = config.model_type == "deepseek" or all( + dim == 0 for dim in (qk_nope_head_dim, qk_rope_head_dim) + ) + self.num_redundant_experts = ( + vllm_config.parallel_config.eplb_config.num_redundant_experts + ) + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -416,199 +696,65 @@ def forward( intermediate_tensors: native.IntermediateTensors | None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | native.IntermediateTensors: - if native.get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - if input_ids is None: - raise ValueError( - "Either input_ids or inputs_embeds must be provided " - "to AFDDeepseekV2Model.forward", - ) - hidden_states = self.embed_input_ids(input_ids) - residual = None - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - residual = intermediate_tensors["residual"] - - llama_4_scaling = self._get_llama_4_scaling(positions) - afd_metadata = get_afd_metadata_from_forward_context() - - aux_hidden_states = [] - if afd_metadata is not None: - if self.aux_hidden_state_layers: - raise RuntimeError( - "AFD DeepSeekV2 E2E wrapper does not support aux hidden " - "state capture yet", - ) - hidden_states, residual = self.forward_with_afd( - hidden_states, - residual, - positions, - afd_metadata, - llama_4_scaling, - ) - else: - for idx, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer), - start=self.start_layer, - ): - if idx in self.aux_hidden_state_layers: - aux_hidden_states.append(hidden_states + residual) - hidden_states, residual = layer( - positions, - hidden_states, - residual, - llama_4_scaling, - ) - - if not native.get_pp_group().is_last_rank: - return native.IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual}, - ) - - hidden_states, _ = self.norm(hidden_states, residual) - if aux_hidden_states: - return hidden_states, aux_hidden_states - return hidden_states - - def forward_with_afd( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - positions: torch.Tensor, - afd_metadata: AFDForwardContextMetadata, - llama_4_scaling: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - if self.afd_config is not None and self.afd_config.compute_gate_on_attention: - forward_context = get_forward_context() - if ( - get_async_moe_ubatch_metadata_from_forward_context(forward_context) - is not None - ): - return self.forward_with_afd_v3( - hidden_states, - residual, - positions, - afd_metadata, - llama_4_scaling, - ) - return self.forward_with_afd_v2( - hidden_states, - residual, - positions, - afd_metadata, - llama_4_scaling, - ) - - afd_connector = afd_metadata.connector - forward_context = get_forward_context() - stage_idx = int( - getattr(forward_context, "ubatch_idx", afd_metadata.stage_idx), - ) - - for layer_offset, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer), - ): - stage_idx = int( - getattr(forward_context, "ubatch_idx", afd_metadata.stage_idx), + if self.afd_config.connector == AFD_ASYNC_CONNECTOR: + from afd_plugin.model_executor.models.npu import ( + deepseek_v2_async_cam_forward, ) - afd_metadata.stage_idx = stage_idx - if layer_offset > 0: - hidden_states = afd_connector.recv_ffn_output( - ref_tensor=hidden_states, - ubatch_idx=stage_idx, - ) - hidden_states, residual = layer( + return deepseek_v2_async_cam_forward.run_model_forward( + self, + input_ids, positions, - hidden_states, - residual, - llama_4_scaling, - ) - metadata = AFDTransferMetadata.create_attention_metadata( - layer_idx=layer.layer_idx, - stage_idx=stage_idx, - seq_len=int(hidden_states.shape[0]), - ) - context = AFDTransferContext(metadata=metadata) - afd_connector.send_attn_output(hidden_states, context) - hidden_states = maybe_apply_dbo_yield( - hidden_states, - role="attention", + intermediate_tensors, + inputs_embeds, ) - - hidden_states = afd_connector.recv_ffn_output( - ref_tensor=hidden_states, - ubatch_idx=stage_idx, + return super().forward( + input_ids, + positions, + intermediate_tensors, + inputs_embeds, ) - return hidden_states, residual - def forward_with_afd_v2( + def compute_ffn_output( self, hidden_states: torch.Tensor, - residual: torch.Tensor | None, - positions: torch.Tensor, - afd_metadata: AFDForwardContextMetadata, - llama_4_scaling: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - from afd_plugin.model_executor.models.npu import ( - deepseek_v2_async_cam_forward, - ) - - return deepseek_v2_async_cam_forward.run_attention_gate_afd_forward( - self, + layer_idx: int, + **kwargs: Any, + ) -> torch.Tensor | AFDF2ATransferPayload: + return self.layers[layer_idx].compute_ffn_output( hidden_states, - residual, - positions, - afd_metadata, - llama_4_scaling, + **kwargs, ) - def forward_with_afd_v3( + def compute_experts_output( self, hidden_states: torch.Tensor, - residual: torch.Tensor | None, - positions: torch.Tensor, - afd_metadata: AFDForwardContextMetadata, - llama_4_scaling: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - forward_context = get_forward_context() - async_moe_ubatch_metadata = get_async_moe_ubatch_metadata_from_forward_context( - forward_context - ) - if async_moe_ubatch_metadata is None: - return self.forward_with_afd_v2( - hidden_states, - residual, - positions, - afd_metadata, - llama_4_scaling, - ) - from afd_plugin.model_executor.models.npu import ( - deepseek_v2_async_cam_forward, + layer_idx: int, + router_logits: torch.Tensor, + ) -> torch.Tensor: + return self.layers[layer_idx].compute_experts_output( + hidden_states, + router_logits, ) - return deepseek_v2_async_cam_forward.run_async_moe_ubatch_afd_forward( - self, - hidden_states, - residual, - positions, - afd_metadata, - async_moe_ubatch_metadata, - llama_4_scaling, + def get_experts_layer_indices(self) -> tuple[int, ...]: + return tuple( + layer_idx + for layer_idx, layer in enumerate(self.layers) + if isinstance(layer, AFDDeepseekV2DecoderLayer) + and layer.uses_remote_experts ) - def compute_ffn_output( + def get_experts_routing_spec( self, - hidden_states: torch.Tensor, layer_idx: int, - **kwargs: Any, - ) -> torch.Tensor | AFDF2ATransferPayload: - return self.layers[layer_idx].compute_ffn_output( - hidden_states, - **kwargs, + ) -> AFDExpertRoutingSpec: + """Return the static native-router contract for graph capture.""" + layer = self.layers[layer_idx] + gate = layer.mlp.gate + return AFDExpertRoutingSpec( + router_logits_width=int(layer.mlp.n_routed_experts), + router_logits_dtype=gate.out_dtype or gate.weight.dtype, ) def _get_llama_4_scaling( @@ -633,32 +779,10 @@ class AFDDeepseekV2ForCausalLM(native.DeepseekV2ForCausalLM): model_cls = AFDDeepseekV2Model def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - self.afd_config = parse_optional_afd_config(vllm_config, validate=False) - self.afd_role = self.afd_config.role if self.afd_config is not None else None + self.afd_config = parse_afd_config(vllm_config, validate=False) + self.afd_role = self.afd_config.role super().__init__(vllm_config=vllm_config, prefix=prefix) - def set_moe_parameters(self) -> None: - self.expert_weights = [] - self.num_expert_groups = getattr(self.config, "n_group", 1) - self.moe_layers = [] - self.moe_mlp_layers = [] - example_moe = None - for layer in self.model.layers: - if isinstance(layer, native.PPMissingLayer): - continue - if not isinstance(layer, native.DeepseekV2DecoderLayer): - continue - mlp = layer._modules.get("mlp") - if (self.afd_role is None or self.afd_role == "ffn") and isinstance( - mlp, native.DeepseekV2MoE - ): - example_moe = mlp - self.moe_mlp_layers.append(mlp) - self.moe_layers.append(mlp.experts) - if self.afd_role == "attention": - return - self.extract_moe_parameters(example_moe) - def compute_ffn_output( self, hidden_states: torch.Tensor, @@ -667,244 +791,35 @@ def compute_ffn_output( ) -> torch.Tensor | AFDF2ATransferPayload: return self.model.compute_ffn_output(hidden_states, layer_idx, **kwargs) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - ascend_config = get_ascend_config() if get_ascend_config is not None else None - stacked_params_mapping = [ - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ("fused_qkv_a_proj", "q_a_proj", 0), - ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), - ] - - mix_placement = ( - getattr(ascend_config, "mix_placement", False) if ascend_config else False - ) - - if self.afd_role == "attention": - vllm_config = get_current_vllm_config() - num_redundant_experts = ( - vllm_config.parallel_config.eplb_config.num_redundant_experts - ) - else: - num_redundant_experts = self.num_redundant_experts - - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts - + (self.config.n_shared_experts if mix_placement else 0), - num_redundant_experts=num_redundant_experts, - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - if ( - self.afd_role == "attention" - and self.afd_config is not None - and self.afd_config.compute_gate_on_attention - and ( - "mlp.gate.weight" in name - or "mlp.gate.e_score_correction_bias" in name - ) - ): - mapped_name = name.replace(".mlp.gate", ".gate") - if mapped_name in params_dict: - param = params_dict[mapped_name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(mapped_name) - continue - - if ( - self.afd_role == "attention" - and self.is_moe_weight(name) - and ( - not self.afd_config.compute_gate_on_attention - or self.is_moe_layer_weight(name) - ) - ): - continue - - if ( - self.afd_role == "ffn" - and self.afd_config.compute_gate_on_attention - and self.is_dense_mlp_weight(name) - ): - continue - - spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) - if spec_layer is not None: - continue - - is_fuse_shared_experts_layer = mix_placement and ( - "mlp.shared_experts" in name - ) - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if ("mlp.experts." in name) and name not in params_dict: - continue - if is_fuse_shared_experts_layer: - continue - name_mapped = name.replace(weight_name, param_name) - - if ( - param_name == "fused_qkv_a_proj" - ) and name_mapped not in params_dict: - continue - else: - name = name_mapped - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - num_chunks = 1 - if is_fuse_shared_experts_layer: - num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 - split_dim = 1 if "down_proj.weight" in name else 0 - total = loaded_weight.shape[split_dim] - assert total % num_chunks == 0, ( - f"Shared expert weight dim {total} " - f"not divisible by num_chunks {num_chunks}" - ) - chunk_size = total // num_chunks - - for j in range(num_chunks): - chunk_name = name - weight_to_load = loaded_weight - - if is_fuse_shared_experts_layer: - if split_dim == 0: - weight_to_load = loaded_weight[ - j * chunk_size : (j + 1) * chunk_size, : - ] - else: - weight_to_load = loaded_weight[ - :, j * chunk_size : (j + 1) * chunk_size - ] - chunk_name = name.replace( - "mlp.shared_experts", - f"mlp.experts.{self.config.n_routed_experts + j}", - ) - - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in chunk_name: - continue - - is_expert_weight = True - if self.afd_role is not None and self.afd_role == "attention": - continue - name_mapped = chunk_name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - if name_mapped not in params_dict: - continue - param = params_dict[name_mapped] - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - weight_to_load, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - if not is_fuse_shared_experts_layer: - name = name_mapped - else: - loaded_params.add(name_mapped) - break - else: - if ( - self.afd_role == "ffn" - and not self.is_moe_weight(name) - and not self.is_common_weight(name) - ): - continue - if is_expert_weight: - continue - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - if not is_fuse_shared_experts_layer: - loaded_params.add(name) - return loaded_params - - def is_moe_weight(self, name): - return ( - "shared_experts" in name - or "experts" in name - or "gate" in name - or "up" in name - or "down" in name + def compute_experts_output( + self, + hidden_states: torch.Tensor, + layer_idx: int, + router_logits: torch.Tensor, + ) -> torch.Tensor: + return self.model.compute_experts_output( + hidden_states, + layer_idx, + router_logits, ) - def is_moe_layer_weight(self, name: str) -> bool: - layer_idx = self.weight_layer_idx(name) - return layer_idx is not None and _is_moe_layer(self.config, layer_idx) + def get_experts_layer_indices(self) -> tuple[int, ...]: + return self.model.get_experts_layer_indices() - def is_dense_mlp_weight(self, name: str) -> bool: - layer_idx = self.weight_layer_idx(name) - return ( - ".mlp." in name - and layer_idx is not None - and not _is_moe_layer(self.config, layer_idx) - ) + def get_experts_routing_spec( + self, + layer_idx: int, + ) -> AFDExpertRoutingSpec: + return self.model.get_experts_routing_spec(layer_idx) - @staticmethod - def weight_layer_idx(name: str) -> int | None: - parts = name.split(".") - for idx, part in enumerate(parts[:-1]): - if part != "layers": - continue - try: - return int(parts[idx + 1]) - except ValueError: - return None - return None - - def is_common_weight(self, name): - return ( - "lm_head" in name - or "model.norm.weight" in name - or "embed_tokens" in name - or "input_layernorm" in name - or "post_attention_layernorm" in name + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return super().load_weights( + _iter_role_weights( + weights, + role=self.afd_role, + config=self.config, + compute_gate_on_attention=self.afd_config.compute_gate_on_attention, + ) ) diff --git a/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py b/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py index f1f677c7..fae5a67c 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py @@ -10,7 +10,9 @@ from typing import TYPE_CHECKING import torch +from vllm.distributed import get_pp_group from vllm.forward_context import get_forward_context +from vllm.sequence import IntermediateTensors from vllm.v1.worker.ubatch_utils import UBatchSlices from afd_plugin.connectors import ( @@ -18,7 +20,11 @@ AFDTransferContext, AFDTransferMetadata, ) -from afd_plugin.model_executor.models import AsyncMoeUbatchMetadata +from afd_plugin.model_executor.models import ( + AsyncMoeUbatchMetadata, + get_afd_metadata_from_forward_context, + get_async_moe_ubatch_metadata_from_forward_context, +) from afd_plugin.v1.worker.dbo import maybe_apply_dbo_yield if TYPE_CHECKING: @@ -28,6 +34,71 @@ ) +def run_model_forward( + model: AFDDeepseekV2Model, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, +) -> torch.Tensor | IntermediateTensors: + """Run the pinned Model fragment around the AFD-owned async schedule.""" + + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + if input_ids is None: + raise ValueError( + "Either input_ids or inputs_embeds must be provided " + "to AFDDeepseekV2Model.forward", + ) + hidden_states = model.embed_input_ids(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + if model.aux_hidden_state_layers: + raise RuntimeError( + "AFD DeepSeekV2 async CAM does not support aux hidden state capture", + ) + forward_context = get_forward_context() + afd_metadata = get_afd_metadata_from_forward_context(forward_context) + if afd_metadata is None: + raise RuntimeError("async CAM requires AFD forward metadata") + llama_4_scaling = model._get_llama_4_scaling(positions) + async_moe_ubatch_metadata = get_async_moe_ubatch_metadata_from_forward_context( + forward_context + ) + if async_moe_ubatch_metadata is None: + hidden_states, residual = run_attention_gate_afd_forward( + model, + hidden_states, + residual, + positions, + afd_metadata, + llama_4_scaling, + ) + else: + hidden_states, residual = run_async_moe_ubatch_afd_forward( + model, + hidden_states, + residual, + positions, + afd_metadata, + async_moe_ubatch_metadata, + llama_4_scaling, + ) + + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual}, + ) + hidden_states, _ = model.norm(hidden_states, residual) + return hidden_states + + def run_attention_gate_afd_forward( model: AFDDeepseekV2Model, hidden_states: torch.Tensor, @@ -436,4 +507,5 @@ def _restore_forward_context_attr( __all__ = [ "run_async_moe_ubatch_afd_forward", "run_attention_gate_afd_forward", + "run_model_forward", ] diff --git a/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py b/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py index a7c60419..17e4cda1 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py @@ -18,7 +18,12 @@ get_ascend_config = None if TYPE_CHECKING: - from afd_plugin.model_executor.models.deepseek_v2 import AFDDeepseekV2DecoderLayer + from vllm.config import VllmConfig + + from afd_plugin.model_executor.models.deepseek_v2 import ( + AFDDeepseekV2DecoderLayer, + _DeepseekAdapterConfig, + ) def compute_attention_gate_topk( @@ -27,7 +32,26 @@ def compute_attention_gate_topk( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Compute router logits and top-k payloads for Attention-side gate.""" - router_logits, _ = layer.gate(hidden_states) + return compute_gate_topk( + gate=layer.mlp.gate, + vllm_config=layer.vllm_config, + config=layer.config, + top_k=layer.top_k, + hidden_states=hidden_states, + ) + + +def compute_gate_topk( + *, + gate: torch.nn.Module, + vllm_config: VllmConfig, + config: _DeepseekAdapterConfig, + top_k: int, + hidden_states: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute routing payloads for a native-path gate proxy.""" + + router_logits, _ = gate(hidden_states) afd_metadata = get_afd_metadata_from_forward_context() if afd_metadata is None: raise RuntimeError( @@ -36,37 +60,35 @@ def compute_attention_gate_topk( ) afd_connector = afd_metadata.connector mix_placement = bool( - getattr(layer.vllm_config, "additional_config", {}).get( + getattr(vllm_config, "additional_config", {}).get( "mix_placement", False, ), ) num_redundant_experts = ( - layer.vllm_config.parallel_config.eplb_config.num_redundant_experts + vllm_config.parallel_config.eplb_config.num_redundant_experts ) if mix_placement: global_num_experts = ( - layer.config.n_shared_experts - + layer.config.n_routed_experts - + num_redundant_experts + config.n_shared_experts + config.n_routed_experts + num_redundant_experts ) else: - global_num_experts = layer.config.n_routed_experts + num_redundant_experts - routed_scaling_factor = getattr(layer.config, "routed_scaling_factor", 1.0) + global_num_experts = config.n_routed_experts + num_redundant_experts + routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) topk_weights, topk_ids = afd_connector.select_experts( hidden_states=hidden_states, router_logits=router_logits, - top_k=layer.top_k, + top_k=top_k, use_grouped_topk=True, - renormalize=getattr(layer.config, "norm_topk_prob", True), - scoring_func=getattr(layer.config, "scoring_func", "softmax"), - num_expert_group=getattr(layer.config, "n_group", 1), - topk_group=getattr(layer.config, "topk_group", 1), + renormalize=getattr(config, "norm_topk_prob", True), + scoring_func=getattr(config, "scoring_func", "softmax"), + num_expert_group=getattr(config, "n_group", 1), + topk_group=getattr(config, "topk_group", 1), routed_scaling_factor=(routed_scaling_factor if mix_placement else 1.0), - e_score_correction_bias=layer.gate.e_score_correction_bias, + e_score_correction_bias=gate.e_score_correction_bias, mix_placement=mix_placement, num_logical_experts=router_logits.shape[1], - num_shared_experts=layer.config.n_shared_experts, + num_shared_experts=config.n_shared_experts, global_num_experts=global_num_experts, ) if force_balanced_topk_ids_enabled(): diff --git a/afd_plugin/v1/worker/attention_model_runner.py b/afd_plugin/v1/worker/attention_model_runner.py index ea6f2867..8a552f65 100644 --- a/afd_plugin/v1/worker/attention_model_runner.py +++ b/afd_plugin/v1/worker/attention_model_runner.py @@ -4,21 +4,27 @@ from __future__ import annotations -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import replace -from typing import Any +from typing import TYPE_CHECKING, Any +import numpy as np import torch import vllm.v1.worker.gpu_model_runner as gpu_model_runner +from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_world_group, ) from vllm.forward_context import BatchDescriptor, DPMetadata, get_forward_context -from vllm.v1.worker.gpu_model_runner import GPUModelRunner +from vllm.sequence import IntermediateTensors +from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput +from vllm.v1.worker.gpu_model_runner import GPUModelRunner, PerLayerAttnMetadata from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper from vllm.v1.worker.ubatch_utils import ( + UBatchSlices, check_ubatch_thresholds, is_last_ubatch_empty, ) @@ -42,14 +48,21 @@ build_ubatch_dp_metadata_list, ) +if TYPE_CHECKING: + from vllm.v1.core.sched.output import SchedulerOutput + class AFDAttentionModelRunner(GPUModelRunner): """Attention model runner that injects AFD metadata into forward context.""" afd_expected_role = "attention" - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + ): + super().__init__(vllm_config, device) self.afd_config = self.parse_config(self.vllm_config) fail_if_unsupported_ubatching(self.vllm_config) self.afd_cudagraph_policy = validate_cuda_graph_mode( @@ -141,13 +154,12 @@ def _send_dp_metadata( self.connector.control_plane.update_state_from_dp_metadata(payload) self.connector.control_plane.send_dp_metadata_list(payload) - def load_model(self, *args: Any, **kwargs: Any) -> Any: + def load_model(self, load_dummy_weights: bool = False) -> None: use_ubatching = bool(self.vllm_config.parallel_config.use_ubatching) with _use_afd_ubatch_wrapper_during_load(use_ubatching): - result = super().load_model(*args, **kwargs) + super().load_model(load_dummy_weights) if use_ubatching: self._install_afd_ubatch_wrapper() - return result def _install_afd_ubatch_wrapper(self) -> None: if isinstance(self.model, AFDUBatchWrapper): @@ -245,23 +257,94 @@ def _install_afd_metadata_on_forward_context( dp_metadata = self._build_capture_dp_metadata(padded_graph_tokens) self._send_dp_metadata(dp_metadata, ubatch_slices) - def _build_attention_metadata(self, *args: Any, **kwargs: Any) -> Any: - num_tokens = kwargs.get("num_tokens", 0) - ubatch_slices = kwargs.get("ubatch_slices") + def _build_attention_metadata( + self, + num_tokens: int, + num_reqs: int, + max_query_len: int, + num_tokens_padded: int | None = None, + num_reqs_padded: int | None = None, + ubatch_slices: UBatchSlices | None = None, + logits_indices: torch.Tensor | None = None, + use_spec_decode: bool = False, + for_cudagraph_capture: bool = False, + num_scheduled_tokens: dict[str, int] | None = None, + cascade_attn_prefix_lens: list[list[int]] | None = None, + slot_mappings: dict[int, torch.Tensor] | None = None, + ) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]: self._afd_pending_metadata = self._build_afd_metadata( ubatch_slices, int(num_tokens), ) - return super()._build_attention_metadata(*args, **kwargs) + return super()._build_attention_metadata( + num_tokens, + num_reqs, + max_query_len, + num_tokens_padded, + num_reqs_padded, + ubatch_slices, + logits_indices, + use_spec_decode, + for_cudagraph_capture, + num_scheduled_tokens, + cascade_attn_prefix_lens, + slot_mappings, + ) - def _determine_batch_execution_and_padding(self, *args: Any, **kwargs: Any) -> Any: + def _determine_batch_execution_and_padding( + self, + num_tokens: int, + num_reqs: int, + num_scheduled_tokens_np: np.ndarray, + max_num_scheduled_tokens: int, + use_cascade_attn: bool, + allow_microbatching: bool = True, + force_eager: bool = False, + force_uniform_decode: bool | None = None, + force_has_lora: bool | None = None, + force_num_active_loras: int | None = None, + num_encoder_reqs: int = 0, + ) -> tuple[ + CUDAGraphMode, + BatchDescriptor, + bool, + torch.Tensor | None, + CUDAGraphStat | None, + ]: ( cudagraph_mode, batch_descriptor, should_ubatch, num_tokens_across_dp, cudagraph_stats, - ) = super()._determine_batch_execution_and_padding(*args, **kwargs) + ) = super()._determine_batch_execution_and_padding( + num_tokens, + num_reqs, + num_scheduled_tokens_np, + max_num_scheduled_tokens, + use_cascade_attn, + allow_microbatching, + force_eager, + force_uniform_decode, + force_has_lora, + force_num_active_loras, + num_encoder_reqs, + ) + + args = ( + num_tokens, + num_reqs, + num_scheduled_tokens_np, + max_num_scheduled_tokens, + use_cascade_attn, + allow_microbatching, + force_eager, + force_uniform_decode, + force_has_lora, + force_num_active_loras, + num_encoder_reqs, + ) + kwargs: dict[str, Any] = {} # determin if ubatch should be activated. # 1. For dp = 1, vLLM hardcodes `should_ubatch=False`. @@ -336,16 +419,47 @@ def _should_ubatch_single_rank( padded_tokens = batch_descriptor.num_tokens return not is_last_ubatch_empty(num_tokens, padded_tokens, num_ubatches) - def _model_forward(self, *args: Any, **kwargs: Any) -> Any: + def _model_forward( + self, + input_ids: torch.Tensor | None = None, + positions: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **model_kwargs: dict[str, Any], + ) -> Any: forward_context = get_forward_context() self._install_afd_metadata_on_forward_context(forward_context) - return super()._model_forward(*args, **kwargs) + return super()._model_forward( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + **model_kwargs, + ) - def execute_model(self, *args: Any, **kwargs: Any) -> Any: + def execute_model( + self, + scheduler_output: SchedulerOutput, + intermediate_tensors: IntermediateTensors | None = None, + ) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors | None: step_afd_gpu_profiler(self.prof) - return super().execute_model(*args, **kwargs) + return super().execute_model(scheduler_output, intermediate_tensors) - def _dummy_run(self, *args: Any, **kwargs: Any) -> Any: + def _dummy_run( + self, + num_tokens: int, + cudagraph_runtime_mode: CUDAGraphMode | None = None, + force_attention: bool = False, + uniform_decode: bool = False, + allow_microbatching: bool = True, + skip_eplb: bool = False, + is_profile: bool = False, + create_mixed_batch: bool = False, + remove_lora: bool = True, + is_graph_capturing: bool = False, + num_active_loras: int = 0, + profile_seq_lens: int | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: """Run vLLM's DP dummy batch through the AFD model path. vLLM uses ``execute_dummy_batch`` on idle DP ranks while another DP rank @@ -362,17 +476,42 @@ def _dummy_run(self, *args: Any, **kwargs: Any) -> Any: "_afd_is_graph_capturing", False, ) - self._afd_is_graph_capturing = bool( - kwargs.get("is_graph_capturing", False), - ) + self._afd_is_graph_capturing = is_graph_capturing try: with use_afd_metadata_provider(self): - return super()._dummy_run(*args, **kwargs) + return super()._dummy_run( + num_tokens, + cudagraph_runtime_mode, + force_attention, + uniform_decode, + allow_microbatching, + skip_eplb, + is_profile, + create_mixed_batch, + remove_lora, + is_graph_capturing, + num_active_loras, + profile_seq_lens, + ) finally: self._afd_is_graph_capturing = previous_is_graph_capturing self._afd_pending_metadata = previous_metadata - def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: + # Patch reason: native capture does not publish AFD warmup/capture metadata. + # Patch functionality: preserve the upstream warmup/capture flow while + # publishing replayable connector state before formal graph capture. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_model_runner.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 + def _warmup_and_capture( + self, + desc: BatchDescriptor, + cudagraph_runtime_mode: CUDAGraphMode, + profile_seq_lens: int | None = None, + allow_microbatching: bool = False, + num_warmups: int | None = None, + profiler: AbstractContextManager[Any] | None = None, + ): """Mirror vLLM warmup/capture while marking AFD warmup metadata. The native implementation calls ``self._dummy_run`` for warmups and @@ -381,27 +520,13 @@ def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: from graph-capture metadata. """ - names = [ - "desc", - "cudagraph_runtime_mode", - "profile_seq_lens", - "allow_microbatching", - "num_warmups", - ] - values = dict(zip(names, args, strict=False)) - values.update(kwargs) - desc = values.get("desc") - cudagraph_runtime_mode = values.get("cudagraph_runtime_mode") - if desc is None or cudagraph_runtime_mode is None: - return super()._warmup_and_capture(*args, **kwargs) - - num_warmups = values.get("num_warmups") + if profiler is None: + profiler = nullcontext() if num_warmups is None: num_warmups = self.compilation_config.cudagraph_num_of_warmups - allow_microbatching = bool(values.get("allow_microbatching", False)) - profile_seq_lens = values.get("profile_seq_lens") force_attention = cudagraph_runtime_mode == CUDAGraphMode.FULL + # ### PATCH START: expose warmup state to the AFD control plane. previous_is_warmup = bool(self._is_warmup) try: self._is_warmup = True @@ -415,10 +540,13 @@ def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: skip_eplb=True, remove_lora=False, num_active_loras=desc.num_active_loras, + profile_seq_lens=profile_seq_lens, ) finally: self._is_warmup = previous_is_warmup + # ### PATCH END: expose warmup state to the AFD control plane. + # ### PATCH START: publish static AFD state before graph capture. previous_metadata = self._afd_pending_metadata previous_suppress_send = self._afd_suppress_metadata_send previous_is_graph_capturing = self._afd_is_graph_capturing @@ -444,25 +572,43 @@ def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: None, ) self._afd_suppress_metadata_send = True - self._dummy_run( - desc.num_tokens, - cudagraph_runtime_mode=cudagraph_runtime_mode, - uniform_decode=desc.uniform, - allow_microbatching=allow_microbatching, - skip_eplb=True, - remove_lora=False, - num_active_loras=desc.num_active_loras, - is_graph_capturing=True, - profile_seq_lens=profile_seq_lens, - ) + with ( + profiler, + torch.profiler.record_function( + f"capture_{desc.num_tokens}_{cudagraph_runtime_mode.name}" + ), + ): + self._dummy_run( + desc.num_tokens, + cudagraph_runtime_mode=cudagraph_runtime_mode, + uniform_decode=desc.uniform, + allow_microbatching=allow_microbatching, + skip_eplb=True, + remove_lora=False, + num_active_loras=desc.num_active_loras, + is_graph_capturing=True, + profile_seq_lens=profile_seq_lens, + ) finally: self._afd_is_graph_capturing = previous_is_graph_capturing self._afd_suppress_metadata_send = previous_suppress_send self._afd_pending_metadata = previous_metadata - + # ### PATCH END: publish static AFD state before graph capture. + + # Patch reason: AFD owns an additional profiler and connector lifecycle. + # Patch functionality: preserve native GPUModelRunner cleanup, then close + # AFD-owned resources even when native cleanup raises. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_model_runner.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 def shutdown(self) -> None: + # ### PATCH START: extend native shutdown with AFD resource cleanup. stop_afd_gpu_profiler(self.prof) - self.connector.close() + try: + super().shutdown() + finally: + self.connector.close() + # ### PATCH END: extend native shutdown with AFD resource cleanup. def _next_afd_transaction_id(self) -> str: counter = self._afd_transaction_counter @@ -551,6 +697,9 @@ def _batch_execution_values( "allow_microbatching", "force_eager", "force_uniform_decode", + "force_has_lora", + "force_num_active_loras", + "num_encoder_reqs", ] values = dict(zip(names, args, strict=False)) values.update(kwargs) diff --git a/afd_plugin/v1/worker/attention_worker.py b/afd_plugin/v1/worker/attention_worker.py index 9c2b996e..a264a69e 100644 --- a/afd_plugin/v1/worker/attention_worker.py +++ b/afd_plugin/v1/worker/attention_worker.py @@ -4,9 +4,8 @@ from __future__ import annotations -from typing import Any - import torch +from vllm.config import VllmConfig from vllm.v1.worker.gpu_worker import Worker from afd_plugin.model_executor.models.model_utils import get_afd_model_config @@ -22,10 +21,23 @@ class AFDAttentionWorker(Worker): afd_expected_role = "attention" - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def __init__( + self, + vllm_config: VllmConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + is_driver_worker: bool = False, + ): + super().__init__( + vllm_config, + local_rank, + rank, + distributed_init_method, + is_driver_worker, + ) - def init_device(self) -> None: + def init_device(self): """Initialize the native GPU worker and swap in the AFD runner.""" assert_compatible_afd_stack( @@ -36,7 +48,7 @@ def init_device(self) -> None: if self.use_v2_model_runner: raise RuntimeError( "AFD Attention runtime currently supports only the vLLM v1 " - "GPUModelRunner; unset VLLM_USE_V2_MODEL_RUNNER", + "GPUModelRunner; set VLLM_USE_V2_MODEL_RUNNER=0", ) fail_if_unsupported_ubatching(self.vllm_config) diff --git a/afd_plugin/v1/worker/dbo.py b/afd_plugin/v1/worker/dbo.py index 9c5a1997..d2b9bf71 100644 --- a/afd_plugin/v1/worker/dbo.py +++ b/afd_plugin/v1/worker/dbo.py @@ -20,7 +20,8 @@ def maybe_apply_dbo_yield( except ImportError: return tensor - return torch.ops.vllm.manual_dbo_yield(tensor) + torch.ops.vllm.manual_dbo_yield(tensor) + return tensor def register_dbo_yield_custom_op() -> None: @@ -29,19 +30,18 @@ def register_dbo_yield_custom_op() -> None: if _AFD_DBO_YIELD_OP_REGISTERED: return - def afd_manual_dbo_yield_op(x: torch.Tensor) -> torch.Tensor: + def afd_manual_dbo_yield_op(x: torch.Tensor) -> None: _yield_if_dbo_enabled() - return x - def afd_manual_dbo_yield_fake(x: torch.Tensor) -> torch.Tensor: - return x + def afd_manual_dbo_yield_fake(x: torch.Tensor) -> None: + return None try: direct_register_custom_op( op_name="manual_dbo_yield", op_func=afd_manual_dbo_yield_op, fake_impl=afd_manual_dbo_yield_fake, - mutates_args=[], + mutates_args=["x"], ) except RuntimeError as exc: if "already" not in str(exc).lower(): diff --git a/afd_plugin/v1/worker/ffn_model_runner.py b/afd_plugin/v1/worker/ffn_model_runner.py index 868af3f7..c5309d61 100644 --- a/afd_plugin/v1/worker/ffn_model_runner.py +++ b/afd_plugin/v1/worker/ffn_model_runner.py @@ -13,9 +13,11 @@ from vllm.config import update_config as update_vllm_config from vllm.distributed.parallel_state import get_world_group, graph_capture from vllm.forward_context import DPMetadata, get_forward_context, set_forward_context +from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT from vllm.model_executor.model_loader import get_model_loader from vllm.utils.mem_utils import DeviceMemoryProfiler from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin +from vllm.v1.worker.workspace import reset_workspace_manager from afd_plugin.compat.profiler import ( create_afd_gpu_profiler, @@ -155,6 +157,7 @@ def execute_model( self._ffn_forward( dp_metadata_list=dp_metadata_list, is_graph_capturing=is_graph_capturing, + is_warmup=is_warmup, ) return None @@ -163,6 +166,7 @@ def _ffn_forward( *, dp_metadata_list: dict[int, DPMetadata | AFDDPMetadata], is_graph_capturing: bool = False, + is_warmup: bool = False, update_connector_state: bool = True, ) -> torch.Tensor | None: if update_connector_state: @@ -170,16 +174,34 @@ def _ffn_forward( _make_dp_metadata_payload( dp_metadata_list, is_graph_capturing=is_graph_capturing, + is_warmup=is_warmup, ), ) rank_ffn_output = None num_layers = max(int(self.num_layers or 0), 1) + experts_layer_indices = frozenset( + self.model.get_experts_layer_indices(), + ) + layer_indices = ( + tuple(sorted(experts_layer_indices)) + if self.afd_config.compute_gate_on_attention + else tuple(range(num_layers)) + ) stage_ids = sorted(int(stage_idx) for stage_idx in dp_metadata_list) or [0] with _ffn_forward_context(self.vllm_config) as forward_context: - for layer_idx in range(num_layers): + for layer_idx in layer_indices: + uses_remote_experts = layer_idx in experts_layer_indices + routing_spec = ( + self.model.get_experts_routing_spec(layer_idx) + if uses_remote_experts and self.afd_config.compute_gate_on_attention + else None + ) for stage_idx in stage_ids: - payload = self.connector.recv_attn_output(ubatch_idx=stage_idx) + payload = self.connector.recv_attn_output( + ubatch_idx=stage_idx, + routing_spec=routing_spec, + ) hidden_states = payload.hidden_states context = payload.context metadata = context.metadata @@ -191,7 +213,22 @@ def _ffn_forward( ) # type: ignore forward_context.additional_kwargs["afd_metadata"] = metadata _set_moe_layer_index(forward_context, layer_idx) - rank_ffn_output = self._execute_eager_mode(hidden_states, layer_idx) + if ( + uses_remote_experts + and self.afd_config.compute_gate_on_attention + ): + router_logits = payload.router_logits + assert router_logits is not None + rank_ffn_output = self.model.compute_experts_output( + hidden_states, + layer_idx, + router_logits, + ) + else: + rank_ffn_output = self._execute_eager_mode( + hidden_states, + layer_idx, + ) self.connector.send_ffn_output(rank_ffn_output, context) return rank_ffn_output @@ -200,11 +237,7 @@ def _execute_eager_mode( hidden_states: torch.Tensor, layer_idx: int, ) -> torch.Tensor: - model = self.model - compute = getattr(model, "compute_ffn_output", None) - if callable(compute): - return compute(hidden_states, layer_idx) - return hidden_states + return self.model.compute_ffn_output(hidden_states, layer_idx) def update_config(self, overrides: dict[str, Any]) -> None: for config_name, config_overrides in overrides.items(): @@ -286,6 +319,7 @@ def capture_model( self._ffn_forward( dp_metadata_list=dp_metadata_list, is_graph_capturing=False, + is_warmup=True, update_connector_state=False, ) else: @@ -327,9 +361,28 @@ def is_pooling_model(self) -> bool: def get_supported_tasks(self) -> tuple[Any, ...]: return () + # Patch reason: the FFN runner owns GPUModelRunner-equivalent CUDA state + # without inheriting GPUModelRunner's shutdown implementation. + # Patch functionality: mirror the pinned native GPU resource cleanup and + # then close AFD-owned profiler and connector resources. + # Signature: matches GPUModelRunner.shutdown; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_model_runner.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 def shutdown(self) -> None: + # ### PATCH START: release native-equivalent and AFD-owned GPU state. stop_afd_gpu_profiler(self.prof) - self.connector.close() + try: + for graph_info in self._cuda_graphs.values(): + graph_info["graph"].reset() + self._cuda_graphs.clear() + self._graph_memory_pool = None + self.vllm_config.compilation_config.static_forward_context.clear() + self.model = None + _ROPE_DICT.clear() + reset_workspace_manager() + finally: + self.connector.close() + # ### PATCH END: release native-equivalent and AFD-owned GPU state. def _resolve_world_ranks() -> tuple[int, int]: diff --git a/afd_plugin/v1/worker/ffn_worker.py b/afd_plugin/v1/worker/ffn_worker.py index 8a354031..5d52179b 100644 --- a/afd_plugin/v1/worker/ffn_worker.py +++ b/afd_plugin/v1/worker/ffn_worker.py @@ -6,10 +6,12 @@ import logging import threading -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import torch +from vllm.config import VllmConfig from vllm.v1.worker.gpu_worker import Worker +from vllm.v1.worker.worker_base import CompilationTimes from afd_plugin.model_executor.models.model_utils import get_afd_model_config from afd_plugin.v1.worker.attention_model_runner import fail_if_unsupported_ubatching @@ -19,6 +21,7 @@ if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec + from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput logger = logging.getLogger(__name__) @@ -33,13 +36,26 @@ class AFDFFNWorker(Worker): afd_expected_role = "ffn" - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def __init__( + self, + vllm_config: VllmConfig, + local_rank: int, + rank: int, + distributed_init_method: str, + is_driver_worker: bool = False, + ): + super().__init__( + vllm_config, + local_rank, + rank, + distributed_init_method, + is_driver_worker, + ) self._ffn_thread: threading.Thread | None = None self._ffn_shutdown_event: threading.Event | None = None self._ffn_loop_error: BaseException | None = None - def init_device(self) -> None: + def init_device(self): """Initialize the native GPU worker and swap in the FFN runner.""" assert_compatible_afd_stack( @@ -50,7 +66,7 @@ def init_device(self) -> None: if self.use_v2_model_runner: raise RuntimeError( "AFD FFN runtime currently supports only the vLLM v1 " - "GPUModelRunner interface; unset VLLM_USE_V2_MODEL_RUNNER", + "GPUModelRunner interface; set VLLM_USE_V2_MODEL_RUNNER=0", ) fail_if_unsupported_ubatching(self.vllm_config) @@ -76,14 +92,17 @@ def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None: self.model_runner.initialize_afd_connector() self.start_ffn_server_loop() - def compile_or_warm_up_model(self) -> float: + def compile_or_warm_up_model(self) -> CompilationTimes: """FFN workers perform no warmup/capture; model execution is driven entirely by connector metadata. """ - return 0.0 + return CompilationTimes(language_model=0.0, encoder=0.0) - def execute_model(self, scheduler_output: SchedulerOutput) -> None: + def execute_model( + self, + scheduler_output: SchedulerOutput, + ) -> ModelRunnerOutput | AsyncModelRunnerOutput | None: """Fail fast if the default scheduler tries to execute FFN work.""" raise RuntimeError( diff --git a/afd_plugin/v1/worker/ubatch_wrapper.py b/afd_plugin/v1/worker/ubatch_wrapper.py index 66004a9d..6ce39e13 100644 --- a/afd_plugin/v1/worker/ubatch_wrapper.py +++ b/afd_plugin/v1/worker/ubatch_wrapper.py @@ -7,6 +7,7 @@ from __future__ import annotations +from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from typing import Any @@ -24,28 +25,48 @@ class AFDUBatchWrapper(UBatchWrapper): """Thin AFD-aware subclass of vLLM's native ``UBatchWrapper``.""" - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def __init__( + self, + runnable: Callable, + vllm_config: VllmConfig, + runtime_mode: CUDAGraphMode, + device: torch.cuda.device, + ): + super().__init__(runnable, vllm_config, runtime_mode, device) self._afd_context_provider: Any | None = None def configure_afd_context_provider(self, provider: Any) -> None: self._afd_context_provider = provider + # Patch reason: native SM partitioning conflicts with AFD connector work. + # Patch functionality: disable native SM partitioning only for active AFD. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_ubatch_wrapper.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 @staticmethod def _create_sm_control_context( vllm_config: VllmConfig, ) -> AbstractContextManager[None]: + # ### PATCH START: leave all SMs visible to AFD compute and communication. if is_afd_active(vllm_config): return nullcontext() + # ### PATCH END: leave all SMs visible to AFD compute and communication. return UBatchWrapper._create_sm_control_context(vllm_config) - def __call__(self, *args: Any, **kwargs: Any) -> Any: + # Patch reason: native ubatch contexts do not carry AFD transfer metadata. + # Patch functionality: install per-ubatch AFD context and control-plane + # metadata while preserving native capture, replay, and execution behavior. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_ubatch_wrapper.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 + def __call__(self, *args, **kwargs): forward_context = get_forward_context() ubatch_slices = forward_context.ubatch_slices if ubatch_slices is None: return super().__call__(*args, **kwargs) cudagraph_runtime_mode = forward_context.cudagraph_runtime_mode + # ### PATCH START: install AFD metadata before splitting ubatches. parent_additional_kwargs = dict(forward_context.additional_kwargs) if "afd_metadata" not in parent_additional_kwargs: self._install_missing_afd_metadata(forward_context, ubatch_slices) @@ -56,6 +77,7 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: self.vllm_config, ubatch_slices, ) + # ### PATCH END: install AFD metadata before splitting ubatches. if ( num_tokens not in self.cudagraphs @@ -125,20 +147,26 @@ def _install_missing_afd_metadata( ubatch_slices, ) + # Patch reason: native per-ubatch contexts omit AFD transfer metadata. + # Patch functionality: clone the parent AFD context into each native ubatch. + # Signature: matches upstream; no added parameters. + # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_ubatch_wrapper.py + # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 def _make_ubatch_metadata( self, - ubatch_slices: Any, - attn_metadata: Any, - slot_mapping: Any, - input_ids: Any, - positions: Any, - inputs_embeds: Any, - intermediate_tensors: Any, - compute_stream: Any, - dp_metadata: list[DPMetadata | AFDDPMetadata], - batch_descriptor: Any, - cudagraph_runtime_mode: Any, + ubatch_slices, + attn_metadata, + slot_mapping, + input_ids, + positions, + inputs_embeds, + intermediate_tensors, + compute_stream, + dp_metadata, + batch_descriptor, + cudagraph_runtime_mode, ) -> list[UbatchMetadata]: + # ### PATCH START: resolve and validate the parent AFD context. parent_forward_context = get_forward_context() parent_additional_kwargs = dict(parent_forward_context.additional_kwargs) afd_metadata = parent_additional_kwargs.get("afd_metadata") @@ -165,10 +193,12 @@ def _make_ubatch_metadata( "AFDUBatchWrapper requires " "ForwardContext.additional_kwargs['afd_metadata']", ) + # ### PATCH END: resolve and validate the parent AFD context. forward_contexts = [] has_slot_mapping = slot_mapping and isinstance(slot_mapping, list) for idx, _ubatch_slice in enumerate(ubatch_slices): + # ### PATCH START: attach one AFD context to each native ubatch. ubatch_afd_metadata = build_ubatch_afd_metadata( afd_metadata, ubatch_slices, @@ -188,6 +218,7 @@ def _make_ubatch_metadata( ), ), ) + # ### PATCH END: attach one AFD context to each native ubatch. ubatch_ctxs = make_ubatch_contexts( num_micro_batches=len(ubatch_slices), diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index ac7d942c..66467b24 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -184,9 +184,9 @@ renaming, shared-expert placement, and redundant experts. AFD adds role filtering: - Attention loads Attention/common parameters and skips FFN expert parameters. - When gate-on-Attention is enabled, MoE gate weights are remapped from the - checkpoint MLP gate into the Attention-owned gate, and dense MLP parameters - remain loadable for locally executed dense layers. + When gate-on-Attention is enabled, MoE gate weights retain the native + `.mlp.gate` path and are also loadable on Attention, while dense MLP + parameters remain loadable for locally executed dense layers. - FFN loads the MLP/expert and required common parameters and skips unrelated Attention parameters. In gate-on-Attention mode it also skips dense MLP parameters because those layers execute on Attention. @@ -199,8 +199,8 @@ the other role can be omitted from model/accuracy E2E coverage. ## Failure and resource ownership -- Missing `afd_metadata` selects the ordinary upstream-style local forward - path; it is not an implicit connector lookup. +- Missing `afd_metadata` on an AFD path fails explicitly; an AFD model alias is + not an implicit local-forward fallback. - AFD paths that require a connector, top-k payload, group list, or async stage metadata fail when that input is missing. - Unsupported aux-hidden-state capture, non-NPU gate placement, unsupported diff --git a/pyproject.toml b/pyproject.toml index 32c597e2..4a82c38a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [] # Keep vLLM optional so macOS and CPU-only development can run import/config tests. # Install the runtime extra only in environments with a compatible vLLM wheel. [project.optional-dependencies] -vllm = ["vllm==0.19.1"] +vllm = ["vllm==0.26.0"] [project.urls] Homepage = "https://github.com/vllm-project/afd-plugin" diff --git a/tests/unit/compat/patches/test_async_dp_engine.py b/tests/unit/compat/patches/test_async_dp_engine.py index d9474e6a..1e3f72d2 100644 --- a/tests/unit/compat/patches/test_async_dp_engine.py +++ b/tests/unit/compat/patches/test_async_dp_engine.py @@ -38,6 +38,7 @@ def _config( local_engines_only=False, data_parallel_backend="mp", enable_elastic_ep=False, + numa_bind=False, ), kv_transfer_config=None, needs_dp_coordinator=True, @@ -160,6 +161,7 @@ def launch_core_engines( utils_module.CoreEngineProcManager = CoreEngineProcManager utils_module.CoreEngineActorManager = CoreEngineActorManager utils_module.get_engine_client_zmq_addr = lambda *_args: "handshake" + utils_module.get_open_port = lambda: 12345 utils_module.get_open_zmq_ipc_path = lambda: "ipc" utils_module.zmq_socket_ctx = zmq_socket_ctx utils_module.zmq = SimpleNamespace(ROUTER="ROUTER") diff --git a/tests/unit/compat/patches/test_async_dp_forward_context.py b/tests/unit/compat/patches/test_async_dp_forward_context.py index b22c8eb1..30f00321 100644 --- a/tests/unit/compat/patches/test_async_dp_forward_context.py +++ b/tests/unit/compat/patches/test_async_dp_forward_context.py @@ -86,6 +86,7 @@ def create_forward_context( slot_mapping, additional_kwargs, skip_compiled, + is_padding=None, ): return SimpleNamespace( attn_metadata=attn_metadata, @@ -97,6 +98,7 @@ def create_forward_context( slot_mapping=slot_mapping, additional_kwargs=additional_kwargs, skip_compiled=skip_compiled, + is_padding=is_padding, ) forward_module.DPMetadata = DPMetadata diff --git a/tests/unit/compat/patches/test_config_validation.py b/tests/unit/compat/patches/test_config_validation.py index 4230575b..f6b418e3 100644 --- a/tests/unit/compat/patches/test_config_validation.py +++ b/tests/unit/compat/patches/test_config_validation.py @@ -21,7 +21,7 @@ def _install_fake_vllm_config(monkeypatch): vllm_module = types.ModuleType("vllm") - vllm_module.__version__ = "0.19.1" + vllm_module.__version__ = "0.26.0" config_package = types.ModuleType("vllm.config") config_module = types.ModuleType("vllm.config.vllm") engine_package = types.ModuleType("vllm.engine") diff --git a/tests/unit/compat/patches/test_engine_core.py b/tests/unit/compat/patches/test_engine_core.py index 2e5adf0e..fed1f0b5 100644 --- a/tests/unit/compat/patches/test_engine_core.py +++ b/tests/unit/compat/patches/test_engine_core.py @@ -87,11 +87,17 @@ def init_none_hash(_hash_fn): def get_request_block_hasher(block_size, hash_fn): return block_size, hash_fn + def register_all_kvcache_specs(_vllm_config): + return None + + def resolve_kv_cache_block_sizes(_kv_cache_config, _vllm_config): + return 16, 16 + core_module.EngineCore = EngineCore core_module.EngineCoreProc = EngineCoreProc core_module.DPEngineCoreProc = DPEngineCoreProc core_module.EngineShutdownState = _EngineShutdownState - core_module.VLLM_VERSION = "0.19.1" + core_module.VLLM_VERSION = "0.26.0" core_module.logger = logging.getLogger("fake-vllm-core") core_module.logger.info_once = lambda *args, **kwargs: None core_module.envs = SimpleNamespace(VLLM_ELASTIC_EP_SCALE_UP_LAUNCH=False) @@ -102,6 +108,8 @@ def get_request_block_hasher(block_size, hash_fn): core_module.get_hash_fn_by_name = get_hash_fn_by_name core_module.init_none_hash = init_none_hash core_module.get_request_block_hasher = get_request_block_hasher + core_module.register_all_kvcache_specs = register_all_kvcache_specs + core_module.resolve_kv_cache_block_sizes = resolve_kv_cache_block_sizes core_module.freeze_gc_heap = lambda: None core_module.maybe_attach_gc_debug_callback = lambda: None core_module.enable_envs_cache = lambda: None @@ -151,7 +159,11 @@ def shutdown(self): decode_context_parallel_size=1, prefill_context_parallel_size=1, ) - model_config = SimpleNamespace(max_model_len=8, runner_type="generate") + model_config = SimpleNamespace( + max_model_len=8, + runner_type="generate", + is_diffusion=False, + ) def validate_block_size(): cache_config.validated = True @@ -164,6 +176,11 @@ def validate_block_size(): model_config=model_config, speculative_config=None, ec_transfer_config=None, + max_concurrent_batches=1, + compilation_config=SimpleNamespace( + compilation_time=0.0, + encoder_compilation_time=0.0, + ), validate_block_size=validate_block_size, ) diff --git a/tests/unit/connectors/test_p2p_connector.py b/tests/unit/connectors/test_p2p_connector.py index 92a36faa..0dd0dba8 100644 --- a/tests/unit/connectors/test_p2p_connector.py +++ b/tests/unit/connectors/test_p2p_connector.py @@ -327,8 +327,8 @@ def direct_register_custom_op(**kwargs): "afd_p2p_send", "afd_p2p_recv", ] - assert calls[0]["mutates_args"] == ["tensor"] - assert calls[1]["mutates_args"] == ["out"] + assert calls[0]["mutates_args"] == ["ordering_token"] + assert calls[1]["mutates_args"] == ["out", "ordering_token"] assert callable(calls[0]["fake_impl"]) assert callable(calls[1]["fake_impl"]) @@ -347,13 +347,15 @@ def test_p2p_hidden_state_send_uses_registered_custom_op(monkeypatch): ), ) connector.a2e_comm_id = 17 + ordering_token = object() + connector._p2p_ordering_token = ordering_token calls = [] torch_module = types.ModuleType("torch") torch_module.ops = SimpleNamespace( vllm=SimpleNamespace( - afd_p2p_send=lambda tensor, dst, comm_id: ( - calls.append((tensor, dst, comm_id)) or None + afd_p2p_send=lambda tensor, token, dst, comm_id: ( + calls.append((tensor, token, dst, comm_id)) or None ), ), ) @@ -372,7 +374,7 @@ def test_p2p_hidden_state_send_uses_registered_custom_op(monkeypatch): connector.a2e_comm_id, ) - assert calls == [(hidden_states, 1, 17)] + assert calls == [(hidden_states, ordering_token, 1, 17)] assert output is None @@ -390,13 +392,15 @@ def test_p2p_recv_preserves_dynamic_ref_tensor_first_dim(monkeypatch): ), ) connector.e2a_comm_id = 23 + ordering_token = object() + connector._p2p_ordering_token = ordering_token calls = [] torch_module = types.ModuleType("torch") torch_module.ops = SimpleNamespace( vllm=SimpleNamespace( - afd_p2p_recv=lambda tensor, src, comm_id: ( - calls.append((tensor, src, comm_id)) or None + afd_p2p_recv=lambda tensor, token, src, comm_id: ( + calls.append((tensor, token, src, comm_id)) or None ), ), ) @@ -426,7 +430,7 @@ def test_p2p_recv_preserves_dynamic_ref_tensor_first_dim(monkeypatch): ) assert output is ref_tensor - assert calls == [(ref_tensor, 0, 23)] + assert calls == [(ref_tensor, ordering_token, 0, 23)] def test_p2p_recv_single_rank_requires_ref_tensor(): diff --git a/tests/unit/connectors/test_p2p_experts_contract.py b/tests/unit/connectors/test_p2p_experts_contract.py new file mode 100644 index 00000000..37901a20 --- /dev/null +++ b/tests/unit/connectors/test_p2p_experts_contract.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") + +from afd_plugin.config import AFDConfig # noqa: E402 +from afd_plugin.connectors import ( # noqa: E402 + AFDExpertRoutingSpec, + AFDTransferContext, + AFDTransferMetadata, +) +from afd_plugin.connectors.gpu.p2p import P2pNcclAFDConnector # noqa: E402 + + +def _vllm_config(): + return SimpleNamespace( + additional_config={}, + model_config=SimpleNamespace( + dtype=torch.bfloat16, + enforce_eager=True, + hf_config=SimpleNamespace(hidden_size=4, num_hidden_layers=3), + ), + parallel_config=SimpleNamespace( + data_parallel_size=1, + data_parallel_rank=0, + ), + ) + + +def _attention_connector(): + return P2pNcclAFDConnector( + rank=0, + local_rank=0, + vllm_config=_vllm_config(), + afd_config=AFDConfig(role="attention"), + ) + + +def _ffn_connector(*, attention_ranks=1): + return P2pNcclAFDConnector( + rank=0, + local_rank=0, + vllm_config=_vllm_config(), + afd_config=AFDConfig( + role="ffn", + num_attention_ranks=attention_ranks, + num_ffn_ranks=1, + ), + ) + + +def _context(): + return AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=2, + stage_idx=1, + seq_len=2, + ), + ) + + +def test_attention_gate_reuses_hidden_state_send_for_router_logits(monkeypatch): + connector = _attention_connector() + hidden_states = torch.ones((2, 4), dtype=torch.bfloat16) + router_logits = torch.ones((2, 3), dtype=torch.float32) + sent = [] + monkeypatch.setattr( + connector, + "_send_hidden_states", + lambda tensor, *args: sent.append(tensor), + ) + + connector.send_attn_output( + hidden_states, + _context(), + router_logits=router_logits, + ) + + assert sent == [hidden_states, router_logits] + + +def test_ffn_gate_sends_only_hidden_states(monkeypatch): + connector = _attention_connector() + hidden_states = torch.ones((2, 4), dtype=torch.bfloat16) + sent = [] + monkeypatch.setattr( + connector, + "_send_hidden_states", + lambda tensor, *args: sent.append(tensor), + ) + + connector.send_attn_output( + hidden_states, + _context(), + ) + + assert sent == [hidden_states] + + +def test_router_shape_is_validated_before_any_send(monkeypatch): + connector = _attention_connector() + sent = [] + monkeypatch.setattr( + connector, + "_send_hidden_states", + lambda tensor, *args: sent.append(tensor), + ) + + with pytest.raises(ValueError, match="equal token counts"): + connector.send_attn_output( + torch.ones((2, 4), dtype=torch.bfloat16), + _context(), + router_logits=torch.ones((3, 3), dtype=torch.float32), + ) + + assert sent == [] + + +def test_attention_gate_fan_in_preserves_peer_order(monkeypatch): + connector = _ffn_connector(attention_ranks=2) + connector.tensor_metadata_list[1] = SimpleNamespace( + device=torch.device("cpu"), + dtype=torch.bfloat16, + size=torch.Size([4, 4]), + ) + for src in (1, 2): + connector._recv_attn_tensor_metadata_list[(1, src)] = SimpleNamespace( + device=torch.device("cpu"), + dtype=torch.bfloat16, + size=torch.Size([2, 4]), + ) + + hidden_peer_1 = torch.full((2, 4), 1, dtype=torch.bfloat16) + router_peer_1 = torch.full((2, 3), 10, dtype=torch.float32) + hidden_peer_2 = torch.full((2, 4), 2, dtype=torch.bfloat16) + router_peer_2 = torch.full((2, 3), 20, dtype=torch.float32) + received = iter( + [hidden_peer_1, router_peer_1, hidden_peer_2, router_peer_2], + ) + monkeypatch.setattr( + connector, + "_recv_hidden_states", + lambda *args, **kwargs: next(received), + ) + + payload = connector.recv_attn_output( + ubatch_idx=1, + routing_spec=AFDExpertRoutingSpec( + router_logits_width=3, + router_logits_dtype=torch.float32, + ), + ) + + assert torch.equal( + payload.hidden_states, + torch.cat([hidden_peer_1, hidden_peer_2]), + ) + assert payload.context.metadata.layer_idx == 0 + assert payload.context.metadata.stage_idx == 1 + assert payload.context.metadata.seq_lens == [2, 2] + assert torch.equal( + payload.router_logits, + torch.cat([router_peer_1, router_peer_2]), + ) + assert payload.context.states is None + + +def test_ffn_gate_receive_does_not_expect_router_logits(monkeypatch): + connector = _ffn_connector() + connector.tensor_metadata_list[0] = SimpleNamespace( + device=torch.device("cpu"), + dtype=torch.bfloat16, + size=torch.Size([2, 4]), + ) + connector._recv_attn_tensor_metadata_list[(0, 1)] = connector.tensor_metadata_list[ + 0 + ] + hidden_states = torch.ones((2, 4), dtype=torch.bfloat16) + received = [] + + def recv_hidden_states(*args, **kwargs): + received.append((args, kwargs)) + return hidden_states + + monkeypatch.setattr(connector, "_recv_hidden_states", recv_hidden_states) + + payload = connector.recv_attn_output() + + assert len(received) == 1 + assert payload.hidden_states is hidden_states + assert payload.router_logits is None + assert payload.context.states is None diff --git a/tests/unit/model_executor/models/test_deepseek_v2_construction.py b/tests/unit/model_executor/models/test_deepseek_v2_construction.py new file mode 100644 index 00000000..82ae52d7 --- /dev/null +++ b/tests/unit/model_executor/models/test_deepseek_v2_construction.py @@ -0,0 +1,643 @@ +from __future__ import annotations + +import ast +import hashlib +import inspect +from pathlib import Path +from types import SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") +nn = torch.nn + +from vllm.config import CompilationMode # noqa: E402 + +from afd_plugin.config import AFDConfig # noqa: E402 +from afd_plugin.model_executor.models import deepseek_v2 as adapter # noqa: E402 + +CONSTRUCTOR_DIGESTS = { + "AFDDeepseekV2Model": ( + "b2e17233e01c98dce0d6640ad97059ee37f70a07488f9c37c2c1885780a43cd7" + ), + "AFDDeepseekV2DecoderLayer": ( + "f1d0b52f12063de217103f8ff1a716e62b743e1653c4dde5ef608fc39a49a780" + ), +} + + +class _FakeStage(nn.Module): + kind = "stage" + + def __init__(self, calls: dict[str, list[str]], *args, prefix="", **kwargs): + super().__init__() + calls[self.kind].append(prefix) + self.weight = nn.Parameter(torch.empty(1)) + + +def _stage_type(kind: str): + return type(f"Fake{kind.title()}", (_FakeStage,), {"kind": kind}) + + +@pytest.fixture +def construction_env(monkeypatch): + calls = { + "attention": [], + "dense": [], + "gate": [], + "moe": [], + "norm": [], + "stage": [], + } + + def bind(stage_type): + return lambda *args, **kwargs: stage_type(calls, *args, **kwargs) + + attention_type = _stage_type("attention") + dense_type = _stage_type("dense") + moe_type = _stage_type("moe") + gate_type = _stage_type("gate") + norm_type = _stage_type("norm") + + monkeypatch.setattr(adapter.native, "DeepseekAttention", bind(attention_type)) + monkeypatch.setattr(adapter.native, "DeepseekV2Attention", bind(attention_type)) + monkeypatch.setattr( + adapter.native, + "DeepseekV2MLAAttention", + bind(attention_type), + ) + monkeypatch.setattr(adapter.native, "DeepseekV2MLP", bind(dense_type)) + monkeypatch.setattr(adapter.native, "DeepseekV2MoE", bind(moe_type)) + monkeypatch.setattr(adapter, "ReplicatedLinear", bind(gate_type)) + monkeypatch.setattr(adapter.native, "RMSNorm", bind(norm_type)) + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="npu"), + ) + return calls + + +def _vllm_config(*, layer_count: int = 2): + config = SimpleNamespace( + first_k_dense_replace=1, + hidden_act="silu", + hidden_size=8, + intermediate_size=16, + model_type="deepseek", + moe_intermediate_size=8, + moe_layer_freq=1, + n_group=1, + n_routed_experts=4, + n_shared_experts=1, + norm_topk_prob=True, + num_attention_heads=2, + num_experts_per_tok=2, + num_hidden_layers=layer_count, + q_lora_rank=None, + qk_nope_head_dim=0, + qk_rope_head_dim=0, + rms_norm_eps=1e-6, + routed_scaling_factor=1.0, + topk_method="noaux_tc", + v_head_dim=0, + vocab_size=32, + ) + return SimpleNamespace( + cache_config=None, + compilation_config=SimpleNamespace(mode=CompilationMode.NONE), + model_config=SimpleNamespace(hf_config=config, use_mla=False), + parallel_config=SimpleNamespace( + enable_eplb=False, + eplb_config=SimpleNamespace(num_redundant_experts=0), + pipeline_parallel_size=1, + use_sequence_parallel_moe=False, + ), + quant_config=None, + scheduler_config=SimpleNamespace(max_num_batched_tokens=8), + ) + + +def _make_layer( + monkeypatch, + *, + role: str, + layer_idx: int, + attention_gate: bool = False, + vllm_config=None, +): + afd_config = AFDConfig( + role=role, + compute_gate_on_attention=attention_gate, + ) + monkeypatch.setattr( + adapter, + "parse_afd_config", + lambda *_args, **_kwargs: afd_config, + ) + if vllm_config is None: + vllm_config = _vllm_config() + return adapter.AFDDeepseekV2DecoderLayer( + vllm_config, + f"model.layers.{layer_idx}", + ) + + +def _parameter_names(module: nn.Module) -> set[str]: + return {name for name, _ in module.named_parameters()} + + +def _constructor_source(class_name: str) -> str: + source = Path(adapter.__file__).read_text() + source_lines = source.splitlines() + module = ast.parse(source) + class_node = next( + node + for node in module.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + constructor = next( + node + for node in class_node.body + if isinstance(node, ast.FunctionDef) and node.name == "__init__" + ) + return "\n".join( + source_lines[constructor.lineno - 1 : constructor.end_lineno], + ) + + +def _masked_patch_sha256(source: str) -> str: + masked_lines = [] + in_patch = False + for line in source.splitlines(): + if "# ### PATCH START" in line: + assert not in_patch + indentation = line[: len(line) - len(line.lstrip())] + masked_lines.append(f"{indentation}# ") + in_patch = True + elif "# ### PATCH END" in line: + assert in_patch + in_patch = False + elif not in_patch: + masked_lines.append(line.rstrip()) + assert not in_patch + masked_source = "\n".join(masked_lines).strip() + "\n" + return hashlib.sha256(masked_source.encode()).hexdigest() + + +def test_pinned_constructor_signatures_match_native_vllm(): + assert inspect.signature(adapter.AFDDeepseekV2Model.__init__) == inspect.signature( + adapter.native.DeepseekV2Model.__init__, + ) + assert inspect.signature( + adapter.AFDDeepseekV2DecoderLayer.__init__, + ) == inspect.signature(adapter.native.DeepseekV2DecoderLayer.__init__) + + +@pytest.mark.parametrize( + ("class_name", "expected_digest"), + CONSTRUCTOR_DIGESTS.items(), +) +def test_non_patch_constructor_source_matches_pinned_vllm( + class_name: str, + expected_digest: str, +) -> None: + assert _masked_patch_sha256(_constructor_source(class_name)) == expected_digest + + +def test_standard_attention_constructs_no_ffn_parameters( + monkeypatch, + construction_env, +): + dense = _make_layer(monkeypatch, role="attention", layer_idx=0) + moe = _make_layer(monkeypatch, role="attention", layer_idx=1) + + assert construction_env["attention"] == [ + "model.layers.0.self_attn", + "model.layers.1.self_attn", + ] + assert construction_env["dense"] == [] + assert construction_env["moe"] == [] + assert isinstance(dense.mlp, adapter.RemoteFFNProxy) + assert isinstance(moe.mlp, adapter.RemoteFFNProxy) + assert not any(name.startswith("mlp.") for name in _parameter_names(dense)) + assert not any(name.startswith("mlp.") for name in _parameter_names(moe)) + + +def test_attention_gate_keeps_dense_local_and_gate_at_mlp_path( + monkeypatch, + construction_env, +): + dense = _make_layer( + monkeypatch, + role="attention", + layer_idx=0, + attention_gate=True, + ) + moe = _make_layer( + monkeypatch, + role="attention", + layer_idx=1, + attention_gate=True, + ) + + assert construction_env["dense"] == ["model.layers.0.mlp"] + assert construction_env["moe"] == [] + assert construction_env["gate"] == ["model.layers.1.mlp.gate"] + assert isinstance(moe.mlp, adapter.GateOnlyRemoteMoE) + assert "mlp.gate.weight" in _parameter_names(moe) + assert "mlp.gate.e_score_correction_bias" in _parameter_names(moe) + assert not any("experts" in name for name in _parameter_names(moe)) + assert not isinstance(dense.mlp, adapter.RemoteFFNProxy) + + +def test_cuda_attention_gate_uses_v026_native_gate_contract( + monkeypatch, + construction_env, +): + gate_calls = [] + + class _FakeGate(nn.Module): + def __init__(self, input_size, output_size, **kwargs): + super().__init__() + gate_calls.append((input_size, output_size, kwargs)) + self.weight = nn.Parameter(torch.empty(output_size, input_size)) + + vllm_config = _vllm_config() + vllm_config.model_config.hf_config.moe_router_dtype = "float32" + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="cuda"), + ) + monkeypatch.setattr(adapter.native, "GateLinear", _FakeGate) + monkeypatch.setattr( + adapter.native, + "get_tensor_model_parallel_world_size", + lambda: 1, + ) + monkeypatch.setattr( + adapter.native, + "get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + adapter.native, + "get_ep_group", + lambda: SimpleNamespace( + device_group=SimpleNamespace(size=lambda: 1), + rank_in_group=0, + ), + ) + + moe = _make_layer( + monkeypatch, + role="attention", + layer_idx=1, + attention_gate=True, + vllm_config=vllm_config, + ) + + assert isinstance(moe.mlp, adapter.AFDDeepseekV2RemoteExpertsMoE) + assert isinstance(moe.mlp.experts, adapter.AFDAttentionFusedMoE) + assert gate_calls == [ + ( + 8, + 4, + { + "out_dtype": torch.float32, + "prefix": "model.layers.1.mlp.gate", + }, + ), + ] + assert list(moe.mlp.experts.parameters()) == [] + assert list(moe.mlp.experts.buffers()) == [] + + +def test_cuda_remote_experts_reject_eplb_on_attention( + monkeypatch, + construction_env, +): + vllm_config = _vllm_config() + vllm_config.parallel_config.enable_eplb = True + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="cuda"), + ) + + with pytest.raises(RuntimeError, match="do not support EPLB"): + _make_layer( + monkeypatch, + role="attention", + layer_idx=1, + attention_gate=True, + vllm_config=vllm_config, + ) + + +def test_cuda_ffn_gate_uses_parameter_free_internal_router_shell( + monkeypatch, + construction_env, +): + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="cuda"), + ) + monkeypatch.setattr( + adapter.native, + "GateLinear", + lambda *_args, **_kwargs: pytest.fail( + "FFN-side gate must not construct an Attention gate", + ), + ) + monkeypatch.setattr( + adapter.native, + "get_tensor_model_parallel_world_size", + lambda: 1, + ) + monkeypatch.setattr( + adapter.native, + "get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + adapter.native, + "get_ep_group", + lambda: SimpleNamespace( + device_group=SimpleNamespace(size=lambda: 1), + rank_in_group=0, + ), + ) + + moe = _make_layer( + monkeypatch, + role="attention", + layer_idx=1, + attention_gate=False, + ) + + assert isinstance(moe.mlp, adapter.AFDDeepseekV2RemoteExpertsMoE) + assert isinstance(moe.mlp.experts, adapter.AFDAttentionFusedMoE) + assert moe.mlp.gate is None + assert moe.mlp.experts.is_internal_router + assert "forward" not in type(moe.mlp).__dict__ + assert list(moe.mlp.experts.parameters()) == [] + assert list(moe.mlp.experts.buffers()) == [] + assert not any(name.startswith("mlp.") for name in _parameter_names(moe)) + + +def test_ffn_constructs_no_real_attention( + monkeypatch, + construction_env, +): + dense = _make_layer(monkeypatch, role="ffn", layer_idx=0) + moe = _make_layer(monkeypatch, role="ffn", layer_idx=1) + + assert construction_env["attention"] == [] + assert construction_env["dense"] == ["model.layers.0.mlp"] + assert construction_env["moe"] == ["model.layers.1.mlp"] + assert isinstance(dense.self_attn, adapter.native.PPMissingLayer) + assert isinstance(moe.self_attn, adapter.native.PPMissingLayer) + assert not any(name.startswith("self_attn.") for name in _parameter_names(dense)) + assert not any(name.startswith("self_attn.") for name in _parameter_names(moe)) + + +@pytest.mark.parametrize( + ("aiter_enabled", "apply_routed_scale_to_output"), + [(False, True), (True, False)], +) +def test_ffn_moe_preserves_v026_native_routed_scale_placement( + monkeypatch, + construction_env, + aiter_enabled, + apply_routed_scale_to_output, +): + moe_calls = [] + + class _FakeMoE(nn.Module): + def __init__(self, **kwargs): + super().__init__() + moe_calls.append(kwargs) + self.experts = SimpleNamespace(gate=object()) + + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="cuda"), + ) + monkeypatch.setattr(adapter.native, "DeepseekV2MoE", _FakeMoE) + monkeypatch.setattr( + adapter.native.rocm_aiter_ops, + "is_fused_moe_enabled", + lambda: aiter_enabled, + ) + + moe = _make_layer( + monkeypatch, + role="ffn", + layer_idx=1, + attention_gate=True, + ) + + assert moe_calls[0]["apply_routed_scale_to_output"] is apply_routed_scale_to_output + assert moe.mlp.experts.gate is None + + +def test_ffn_side_gate_keeps_native_internal_router( + monkeypatch, + construction_env, +): + native_gate = object() + + class _FakeMoE(nn.Module): + def __init__(self, **_kwargs): + super().__init__() + self.experts = SimpleNamespace(gate=native_gate) + + monkeypatch.setattr( + adapter.native, + "current_platform", + SimpleNamespace(device_type="cuda"), + ) + monkeypatch.setattr(adapter.native, "DeepseekV2MoE", _FakeMoE) + monkeypatch.setattr( + adapter.native.rocm_aiter_ops, + "is_fused_moe_enabled", + lambda: False, + ) + + moe = _make_layer( + monkeypatch, + role="ffn", + layer_idx=1, + attention_gate=False, + ) + + assert moe.mlp.experts.gate is native_gate + + +def test_attention_gate_ffn_dense_uses_non_executing_placeholder( + monkeypatch, + construction_env, +): + dense = _make_layer( + monkeypatch, + role="ffn", + layer_idx=0, + attention_gate=True, + ) + moe = _make_layer( + monkeypatch, + role="ffn", + layer_idx=1, + attention_gate=True, + ) + + assert construction_env["attention"] == [] + assert construction_env["dense"] == [] + assert construction_env["moe"] == ["model.layers.1.mlp"] + assert isinstance(dense.mlp, adapter.native.PPMissingLayer) + assert isinstance(moe.self_attn, adapter.native.PPMissingLayer) + + +def _patch_model_constructor_dependencies( + monkeypatch, + construction_env, +): + monkeypatch.setattr( + adapter.native, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + monkeypatch.setattr( + adapter.native, + "VocabParallelEmbedding", + lambda *args, **kwargs: _FakeStage(construction_env, *args, **kwargs), + ) + + def make_layers(count, factory, *, prefix): + layers = nn.ModuleList( + factory(f"{prefix}.{layer_idx}") for layer_idx in range(count) + ) + return 0, count, layers + + monkeypatch.setattr(adapter.native, "make_layers", make_layers) + monkeypatch.setattr( + adapter.native, + "make_empty_intermediate_tensors_factory", + lambda *_args, **_kwargs: object(), + ) + monkeypatch.setattr( + adapter.native.DeepseekV2Model, + "__init__", + lambda *_args, **_kwargs: pytest.fail("native constructor was called"), + ) + + +def test_model_constructor_uses_role_aware_layers( + monkeypatch, + construction_env, +): + vllm_config = _vllm_config() + afd_config = AFDConfig(role="attention") + monkeypatch.setattr( + adapter, + "parse_afd_config", + lambda *_args, **_kwargs: afd_config, + ) + _patch_model_constructor_dependencies(monkeypatch, construction_env) + + model = adapter.AFDDeepseekV2Model(vllm_config=vllm_config, prefix="model") + + assert isinstance(model, adapter.native.DeepseekV2Model) + assert all( + isinstance(layer, adapter.AFDDeepseekV2DecoderLayer) for layer in model.layers + ) + assert construction_env["dense"] == [] + assert construction_env["moe"] == [] + assert model.hidden_size == vllm_config.model_config.hf_config.hidden_size + assert model.use_mha + assert model.num_redundant_experts == 0 + assert ( + sum( + base.__name__ == "TorchCompileWithNoGuardsWrapper" + for base in type(model).__mro__ + ) + == 1 + ) + + +@pytest.mark.parametrize( + ("role", "expected_allocations"), + [("attention", 1), ("ffn", 0)], +) +def test_v32_indexer_buffer_is_allocated_only_on_attention( + monkeypatch, + construction_env, + role, + expected_allocations, +): + vllm_config = _vllm_config() + vllm_config.model_config.hf_config.index_topk = 2048 + afd_config = AFDConfig(role=role) + monkeypatch.setattr( + adapter, + "parse_afd_config", + lambda *_args, **_kwargs: afd_config, + ) + _patch_model_constructor_dependencies(monkeypatch, construction_env) + + original_empty = torch.empty + indexer_allocations = [] + + def track_empty(*size, **kwargs): + if size[:2] == (8, 2048): + indexer_allocations.append((size, kwargs.copy())) + kwargs = {key: value for key, value in kwargs.items() if key != "device"} + return original_empty(*size, **kwargs) + + monkeypatch.setattr(adapter.torch, "empty", track_empty) + + model = adapter.AFDDeepseekV2Model(vllm_config=vllm_config, prefix="model") + + assert model.is_v32 + assert len(indexer_allocations) == expected_allocations + + +def test_afd_alias_without_activation_fails_before_construction(monkeypatch): + vllm_config = SimpleNamespace( + additional_config={}, + compilation_config=SimpleNamespace(mode=CompilationMode.NONE), + ) + + with pytest.raises(ValueError, match="requires additional_config"): + adapter.AFDDeepseekV2Model(vllm_config=vllm_config, prefix="model") + with pytest.raises(ValueError, match="requires additional_config"): + adapter.AFDDeepseekV2DecoderLayer(vllm_config, "model.layers.0") + + assert issubclass( + adapter.AFDDeepseekV2ForCausalLM, + adapter.native.DeepseekV2ForCausalLM, + ) + assert adapter.AFDDeepseekV2ForCausalLM.model_cls is adapter.AFDDeepseekV2Model + + +def test_model_constructor_rejects_sequence_parallel_moe_before_allocation( + monkeypatch, + construction_env, +): + vllm_config = _vllm_config() + vllm_config.parallel_config.use_sequence_parallel_moe = True + monkeypatch.setattr( + adapter, + "parse_afd_config", + lambda *_args, **_kwargs: AFDConfig(role="ffn"), + ) + + with pytest.raises(RuntimeError, match="sequence-parallel MoE"): + adapter.AFDDeepseekV2Model(vllm_config=vllm_config, prefix="model") + + assert all(not calls for calls in construction_env.values()) diff --git a/tests/unit/model_executor/models/test_deepseek_v2_proxy.py b/tests/unit/model_executor/models/test_deepseek_v2_proxy.py new file mode 100644 index 00000000..b650f480 --- /dev/null +++ b/tests/unit/model_executor/models/test_deepseek_v2_proxy.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") +nn = torch.nn + +from afd_plugin.config import AFD_ASYNC_CONNECTOR, AFDConfig # noqa: E402 +from afd_plugin.model_executor.models import deepseek_v2 as adapter # noqa: E402 + + +class _FakeConnector: + def __init__(self, events: list[tuple]) -> None: + self.events = events + + def send_attn_output(self, hidden_states, context, **kwargs) -> None: + self.events.append(("send", hidden_states, context, kwargs)) + + def recv_ffn_output(self, *, ref_tensor, ubatch_idx): + self.events.append(("recv", ref_tensor, ubatch_idx)) + return ref_tensor * 0.25 + + +class _PassthroughNorm(nn.Module): + def forward(self, hidden_states, residual=None): + if residual is None: + return hidden_states + return hidden_states, residual + + +class _FakeAttention(nn.Module): + def forward(self, positions, hidden_states): + return hidden_states + + +def _install_fake_forward_context(monkeypatch, events, *, stage_idx=2): + connector = _FakeConnector(events) + afd_metadata = SimpleNamespace(connector=connector, stage_idx=9) + monkeypatch.setattr( + adapter, + "get_afd_metadata_from_forward_context", + lambda: afd_metadata, + ) + monkeypatch.setattr( + adapter, + "get_forward_context", + lambda: SimpleNamespace(ubatch_idx=stage_idx), + ) + + def record_yield(hidden_states, *, role): + events.append(("yield", hidden_states, role)) + return hidden_states + + monkeypatch.setattr(adapter, "maybe_apply_dbo_yield", record_yield) + return afd_metadata + + +@pytest.mark.parametrize("layer_idx", [0, 1], ids=["dense", "moe"]) +def test_native_decoder_forward_calls_remote_proxy_once( + monkeypatch, + layer_idx, +): + events = [] + afd_metadata = _install_fake_forward_context(monkeypatch, events) + monkeypatch.setattr(adapter.native, "DeepseekAttention", _FakeAttention) + + layer = object.__new__(adapter.AFDDeepseekV2DecoderLayer) + nn.Module.__init__(layer) + layer.layer_idx = layer_idx + layer.use_mha = True + layer.use_sequence_parallel_moe = False + layer.routed_scaling_factor = 4.0 + layer.input_layernorm = _PassthroughNorm() + layer.self_attn = _FakeAttention() + layer.post_attention_layernorm = _PassthroughNorm() + layer.mlp = adapter.RemoteFFNProxy(layer_idx=layer_idx) + + hidden_states = torch.full((2, 4), 8.0, dtype=torch.float16) + output, residual = layer( + torch.arange(2), + hidden_states, + None, + ) + + assert [event[0] for event in events] == ["send", "yield", "recv"] + sent_metadata = events[0][2].metadata + assert sent_metadata.layer_idx == layer_idx + assert sent_metadata.stage_idx == 2 + assert sent_metadata.seq_lens == [2] + assert events[1][2] == "attention" + assert events[2][2] == 2 + assert afd_metadata.stage_idx == 2 + assert torch.equal(output, hidden_states * 0.25) + assert torch.equal(residual, hidden_states) + + +@pytest.mark.parametrize( + ("mlp_type", "expected_scale"), + [("dense", 0.25), ("moe", 1.0)], +) +def test_ffn_compute_applies_dense_fp16_scaling_once( + monkeypatch, + mlp_type, + expected_scale, +): + class FakeDenseMLP(nn.Module): + def forward(self, hidden_states): + return hidden_states.clone() + + class FakeMoE(nn.Module): + def forward(self, hidden_states): + return hidden_states.clone() + + monkeypatch.setattr(adapter.native, "DeepseekV2MLP", FakeDenseMLP) + layer = object.__new__(adapter.AFDDeepseekV2DecoderLayer) + nn.Module.__init__(layer) + layer.compute_gate_on_attention = False + layer.routed_scaling_factor = 4.0 + layer.mlp = FakeDenseMLP() if mlp_type == "dense" else FakeMoE() + hidden_states = torch.full((2, 4), 8.0, dtype=torch.float16) + + output = layer.compute_ffn_output(hidden_states) + + assert torch.equal(output, hidden_states * expected_scale) + + +def test_gate_proxy_sends_routing_payload(monkeypatch): + from afd_plugin.model_executor.models.npu import deepseek_v2_attention_gate + + events = [] + _install_fake_forward_context(monkeypatch, events, stage_idx=1) + topk_weights = torch.tensor([[0.75, 0.25]]) + topk_ids = torch.tensor([[1, 3]]) + router_logits = torch.tensor([[0.1, 0.2, 0.3, 0.4]]) + gate_calls = [] + + def compute_gate_topk(**kwargs): + gate_calls.append(kwargs) + return topk_weights, topk_ids, router_logits + + monkeypatch.setattr( + deepseek_v2_attention_gate, + "compute_gate_topk", + compute_gate_topk, + ) + proxy = object.__new__(adapter.GateOnlyRemoteMoE) + adapter.RemoteFFNProxy.__init__(proxy, layer_idx=3) + proxy.gate = nn.Linear(4, 4, bias=False) + proxy.vllm_config = object() + proxy.config = object() + proxy.top_k = 2 + hidden_states = torch.ones(1, 4) + + output = proxy(hidden_states) + + assert len(gate_calls) == 1 + assert gate_calls[0]["gate"] is proxy.gate + assert [event[0] for event in events] == ["send", "yield", "recv"] + send_kwargs = events[0][3] + assert send_kwargs["router_logits"] is router_logits + assert send_kwargs["topk_ids"] is topk_ids + assert send_kwargs["topk_weights"] is topk_weights + assert torch.equal(output, hidden_states * 0.25) + + +def test_remote_experts_proxy_sends_router_logits(monkeypatch): + events = [] + _install_fake_forward_context(monkeypatch, events, stage_idx=1) + proxy = adapter.AFDAttentionFusedMoE( + layer_idx=3, + is_internal_router=False, + ) + hidden_states = torch.ones(1, 4) + router_logits = torch.ones(1, 8) + + output = proxy(hidden_states, router_logits) + + assert [event[0] for event in events] == ["send", "yield", "recv"] + context = events[0][2] + assert context.metadata.layer_idx == 3 + assert context.metadata.stage_idx == 1 + assert context.states is None + assert events[0][3]["router_logits"] is router_logits + assert torch.equal(output, hidden_states * 0.25) + + +def test_remote_proxy_requires_forward_metadata(monkeypatch): + monkeypatch.setattr( + adapter, + "get_afd_metadata_from_forward_context", + lambda: None, + ) + + with pytest.raises(RuntimeError, match="requires AFD forward metadata"): + adapter.RemoteFFNProxy(layer_idx=0)(torch.ones(1, 4)) + + +def test_synchronous_model_forward_delegates_to_native(monkeypatch): + calls = [] + expected = torch.ones(1, 4) + + def native_forward(instance, *args): + calls.append((instance, args)) + return expected + + monkeypatch.setattr(adapter.native.DeepseekV2Model, "forward", native_forward) + model = object.__new__(adapter.AFDDeepseekV2Model) + nn.Module.__init__(model) + model.afd_config = AFDConfig(role="attention") + positions = torch.arange(1) + + output = adapter.AFDDeepseekV2Model.forward( + model, + None, + positions, + None, + ) + + assert output is expected + assert calls == [(model, (None, positions, None, None))] + + +def test_async_connector_dispatches_to_schedule_adapter(monkeypatch): + from afd_plugin.model_executor.models.npu import deepseek_v2_async_cam_forward + + expected = torch.ones(1, 4) + calls = [] + + def async_forward(*args): + calls.append(args) + return expected + + monkeypatch.setattr( + deepseek_v2_async_cam_forward, + "run_model_forward", + async_forward, + ) + monkeypatch.setattr( + adapter.native.DeepseekV2Model, + "forward", + lambda *_args: pytest.fail("native synchronous forward was called"), + ) + model = object.__new__(adapter.AFDDeepseekV2Model) + nn.Module.__init__(model) + model.afd_config = AFDConfig( + role="attention", + connector=AFD_ASYNC_CONNECTOR, + ) + positions = torch.arange(1) + + output = adapter.AFDDeepseekV2Model.forward( + model, + None, + positions, + None, + ) + + assert output is expected + assert calls == [(model, None, positions, None, None)] + + +def test_decoder_inherits_native_forward_without_override(): + assert "forward" not in adapter.AFDDeepseekV2DecoderLayer.__dict__ + assert ( + adapter.AFDDeepseekV2DecoderLayer.forward + is adapter.native.DeepseekV2DecoderLayer.forward + ) diff --git a/tests/unit/model_executor/models/test_deepseek_v2_weight_policy.py b/tests/unit/model_executor/models/test_deepseek_v2_weight_policy.py new file mode 100644 index 00000000..b1aab07c --- /dev/null +++ b/tests/unit/model_executor/models/test_deepseek_v2_weight_policy.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") + +from vllm.model_executor.models import deepseek_v2 as native # noqa: E402 + +from afd_plugin.model_executor.models.deepseek_v2 import ( # noqa: E402 + AFDDeepseekV2ForCausalLM, + AFDDeepseekV2Model, + _checkpoint_weight_roles, +) + +REPO_ROOT = Path(__file__).resolve().parents[4] +ATTENTION_ROLES = frozenset(("attention",)) +FFN_ROLES = frozenset(("ffn",)) +BOTH_ROLES = frozenset(("attention", "ffn")) +WEIGHT_ROLE_GROUPS = ( + ( + BOTH_ROLES, + BOTH_ROLES, + ( + ("embedding", "model.embed_tokens.weight"), + ("final-norm", "model.norm.weight"), + ("lm-head", "lm_head.weight"), + ("decoder-norm", "model.layers.0.input_layernorm.weight"), + ), + ), + ( + ATTENTION_ROLES, + ATTENTION_ROLES, + ( + ("mha-q-projection", "model.layers.0.self_attn.q_proj.weight"), + ("mha-k-projection", "model.layers.0.self_attn.k_proj.weight"), + ("mha-v-projection", "model.layers.0.self_attn.v_proj.weight"), + ("mla-q-a-projection", "model.layers.0.self_attn.q_a_proj.weight"), + ( + "mla-kv-a-projection", + "model.layers.0.self_attn.kv_a_proj_with_mqa.weight", + ), + ("indexer-projection", "model.layers.3.self_attn.indexer.wq_b.weight"), + ("attention-kv-scale", "model.layers.3.self_attn.attn.k_scale"), + ), + ), + ( + FFN_ROLES, + ATTENTION_ROLES, + ( + ("dense-gate-projection", "model.layers.0.mlp.gate_proj.weight"), + ("dense-up-projection", "model.layers.0.mlp.up_proj.weight"), + ("dense-down-projection", "model.layers.0.mlp.down_proj.weight"), + ), + ), + ( + FFN_ROLES, + BOTH_ROLES, + ( + ("moe-gate", "model.layers.3.mlp.gate.weight"), + ("moe-gate-bias", "model.layers.3.mlp.gate.e_score_correction_bias"), + ), + ), + ( + FFN_ROLES, + FFN_ROLES, + ( + ( + "moe-expert-projection", + "model.layers.3.mlp.experts.0.gate_proj.weight", + ), + ( + "moe-expert-scale", + "model.layers.3.mlp.experts.0.gate_proj.weight_scale_inv", + ), + ( + "shared-expert-projection", + "model.layers.3.mlp.shared_experts.gate_proj.weight", + ), + ), + ), +) +WEIGHT_ROLE_CASES = tuple( + (case_id, checkpoint_name, standard_roles, attention_gate_roles) + for standard_roles, attention_gate_roles, cases in WEIGHT_ROLE_GROUPS + for case_id, checkpoint_name in cases +) + + +def _config() -> SimpleNamespace: + return SimpleNamespace( + first_k_dense_replace=3, + moe_layer_freq=1, + model_type="deepseek", + n_routed_experts=64, + n_shared_experts=1, + num_nextn_predict_layers=0, + ) + + +class _OneShotWeights: + def __init__(self, names: list[str]) -> None: + self.items = [(name, torch.tensor([index])) for index, name in enumerate(names)] + self.iterations = 0 + + def __iter__(self): + self.iterations += 1 + if self.iterations > 1: + raise AssertionError("checkpoint iterator was consumed more than once") + return iter(self.items) + + +@pytest.mark.parametrize( + ("checkpoint_name", "standard_roles", "attention_gate_roles"), + [case[1:] for case in WEIGHT_ROLE_CASES], + ids=[case[0] for case in WEIGHT_ROLE_CASES], +) +def test_weight_role_policy( + checkpoint_name: str, + standard_roles: frozenset[str], + attention_gate_roles: frozenset[str], +) -> None: + assert ( + _checkpoint_weight_roles( + checkpoint_name, + _config(), + compute_gate_on_attention=False, + ) + == standard_roles + ) + assert ( + _checkpoint_weight_roles( + checkpoint_name, + _config(), + compute_gate_on_attention=True, + ) + == attention_gate_roles + ) + + +def test_load_weights_passes_one_shot_generator_to_native_loader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + names = [ + "model.embed_tokens.weight", + "model.layers.0.self_attn.q_proj.weight", + "model.layers.0.mlp.gate_proj.weight", + "model.layers.3.mlp.experts.0.down_proj.weight", + ] + weights = _OneShotWeights(names) + seen: list[str] = [] + native_result = {"native.loaded_params"} + + def fake_native_loader(self, filtered_weights): + assert iter(filtered_weights) is filtered_weights + seen.extend(name for name, _ in filtered_weights) + return native_result + + monkeypatch.setattr( + native.DeepseekV2ForCausalLM, + "load_weights", + fake_native_loader, + ) + model = object.__new__(AFDDeepseekV2ForCausalLM) + object.__setattr__(model, "afd_role", "attention") + object.__setattr__( + model, + "afd_config", + SimpleNamespace(compute_gate_on_attention=False), + ) + object.__setattr__(model, "config", _config()) + + result = model.load_weights(weights) + + assert result is native_result + assert seen == names[:2] + assert weights.iterations == 1 + + +def test_native_model_loader_packs_mha_qkv( + monkeypatch: pytest.MonkeyPatch, +) -> None: + target_name = "layers.0.self_attn.qkv_proj.weight" + loaded_shards: list[tuple[str, torch.Tensor]] = [] + parameter = torch.nn.Parameter(torch.zeros(1)) + + def load_qkv(param, loaded_weight, shard_id): + assert param is parameter + loaded_shards.append((shard_id, loaded_weight)) + + parameter.weight_loader = load_qkv + monkeypatch.setattr( + native.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: False, + ) + monkeypatch.setattr( + native, + "fused_moe_make_expert_params_mapping", + lambda *args, **kwargs: [], + ) + monkeypatch.setattr(native, "get_pp_missing_layer_names", lambda *args: set()) + monkeypatch.setattr(native, "is_pp_missing_parameter", lambda *args: False) + + model = object.__new__(AFDDeepseekV2Model) + object.__setattr__(model, "config", _config()) + object.__setattr__(model, "use_mha", True) + object.__setattr__(model, "num_redundant_experts", 0) + object.__setattr__( + model, + "named_parameters", + lambda: iter(((target_name, parameter),)), + ) + qkv_weights = [ + (f"layers.0.self_attn.{projection}_proj.weight", torch.tensor([i])) + for i, projection in enumerate(("q", "k", "v")) + ] + + loaded_params = model.load_weights(iter(qkv_weights)) + + assert loaded_params == {target_name} + assert [shard_id for shard_id, _ in loaded_shards] == ["q", "k", "v"] + assert [weight.item() for _, weight in loaded_shards] == [0, 1, 2] + + +def test_moe_metadata_and_backend_loading_remain_native_owned() -> None: + assert "set_moe_parameters" not in AFDDeepseekV2ForCausalLM.__dict__ + source = ( + REPO_ROOT / "afd_plugin" / "model_executor" / "models" / "deepseek_v2.py" + ).read_text(encoding="utf-8") + assert "vllm_ascend" not in source diff --git a/tests/unit/model_executor/models/test_forward_context.py b/tests/unit/model_executor/models/test_forward_context.py index a7d4e776..b863470c 100644 --- a/tests/unit/model_executor/models/test_forward_context.py +++ b/tests/unit/model_executor/models/test_forward_context.py @@ -5,9 +5,10 @@ import pytest +torch = pytest.importorskip("torch") pytest.importorskip("vllm") -from afd_plugin.model_executor.models import ( +from afd_plugin.model_executor.models import ( # noqa: E402 ASYNC_MOE_UBATCH_METADATA_KEY, get_afd_metadata_from_forward_context, get_async_moe_ubatch_metadata_from_forward_context, @@ -43,6 +44,211 @@ def test_get_async_moe_ubatch_metadata_from_additional_kwargs(): ) +@pytest.mark.parametrize("raise_inside", [False, True]) +def test_async_moe_ubatch_forward_context_restores_state(raise_inside): + from afd_plugin.model_executor.models.npu import ( + deepseek_v2_async_cam_forward as async_forward, + ) + + class CloneableMetadata(SimpleNamespace): + def clone(self): + return CloneableMetadata(**vars(self)) + + parent_metadata = CloneableMetadata( + stage_idx=7, + num_stages=1, + tokens_unpadded_lens=[2, 3], + ) + ubatch_slices = [ + SimpleNamespace( + token_slice=slice(0, 2), + request_slice=slice(0, 1), + num_tokens=2, + ), + SimpleNamespace( + token_slice=slice(2, 5), + request_slice=slice(1, 2), + num_tokens=3, + ), + ] + async_metadata = { + "ubatch_slices": ubatch_slices, + "attn_metadata": ["attention-0", "attention-1"], + } + original_kwargs = { + "afd_metadata": parent_metadata, + "preserved": object(), + } + forward_context = SimpleNamespace( + attn_metadata="parent-attention", + additional_kwargs=original_kwargs, + ubatch_idx=7, + num_tokens=5, + ) + + def run_context(): + with async_forward._use_async_moe_ubatch_forward_context( + forward_context=forward_context, + parent_afd_metadata=parent_metadata, + async_moe_ubatch_metadata=async_metadata, + stage_idx=1, + ): + assert forward_context.attn_metadata == "attention-1" + assert forward_context.ubatch_idx == 1 + assert forward_context.num_ubatches == 2 + assert forward_context.num_tokens == 3 + assert forward_context.additional_kwargs is not original_kwargs + assert ( + forward_context.additional_kwargs["preserved"] + is original_kwargs["preserved"] + ) + stage_metadata = forward_context.additional_kwargs["afd_metadata"] + assert stage_metadata is not parent_metadata + assert stage_metadata.stage_idx == 1 + assert stage_metadata.tokens_start_loc == [2] + assert stage_metadata.requests_start_loc == [1] + assert stage_metadata.tokens_lens == [3] + assert stage_metadata.tokens_unpadded_lens == [3] + if raise_inside: + raise RuntimeError("test failure") + + if raise_inside: + with pytest.raises(RuntimeError, match="test failure"): + run_context() + else: + run_context() + + assert forward_context.attn_metadata == "parent-attention" + assert forward_context.additional_kwargs is original_kwargs + assert forward_context.ubatch_idx == 7 + assert forward_context.num_tokens == 5 + assert not hasattr(forward_context, "num_ubatches") + + +@pytest.mark.parametrize( + ("is_first_rank", "is_last_rank"), + [(True, False), (False, True)], +) +def test_async_model_forward_preserves_pp_boundaries( + monkeypatch, + is_first_rank, + is_last_rank, +): + from afd_plugin.model_executor.models.npu import ( + deepseek_v2_async_cam_forward as async_forward, + ) + + class FakeIntermediateTensors(dict): + pass + + monkeypatch.setattr(async_forward, "IntermediateTensors", FakeIntermediateTensors) + monkeypatch.setattr( + async_forward, + "get_pp_group", + lambda: SimpleNamespace( + is_first_rank=is_first_rank, + is_last_rank=is_last_rank, + ), + ) + forward_context = SimpleNamespace() + afd_metadata = object() + monkeypatch.setattr(async_forward, "get_forward_context", lambda: forward_context) + monkeypatch.setattr( + async_forward, + "get_afd_metadata_from_forward_context", + lambda context: afd_metadata if context is forward_context else None, + ) + monkeypatch.setattr( + async_forward, + "get_async_moe_ubatch_metadata_from_forward_context", + lambda context: None, + ) + + schedule_calls = [] + + def run_schedule( + model, + hidden_states, + residual, + positions, + received_metadata, + llama_4_scaling, + ): + schedule_calls.append( + ( + model, + hidden_states, + residual, + positions, + received_metadata, + llama_4_scaling, + ) + ) + next_residual = ( + torch.zeros_like(hidden_states) if residual is None else residual + 2 + ) + return hidden_states + 1, next_residual + + monkeypatch.setattr( + async_forward, + "run_attention_gate_afd_forward", + run_schedule, + ) + norm_calls = [] + + def run_norm(hidden_states, residual): + norm_calls.append((hidden_states, residual)) + return hidden_states + residual, None + + model = SimpleNamespace( + aux_hidden_state_layers=(), + embed_input_ids=lambda input_ids: input_ids.to(torch.float32).unsqueeze(-1), + _get_llama_4_scaling=lambda positions: None, + norm=run_norm, + ) + positions = torch.arange(2) + if is_first_rank: + input_ids = torch.tensor([3, 4]) + intermediate_tensors = None + expected_hidden_states = model.embed_input_ids(input_ids) + expected_residual = None + else: + input_ids = None + expected_hidden_states = torch.full((2, 1), 5.0) + expected_residual = torch.full((2, 1), 7.0) + intermediate_tensors = FakeIntermediateTensors( + { + "hidden_states": expected_hidden_states, + "residual": expected_residual, + } + ) + + output = async_forward.run_model_forward( + model, + input_ids, + positions, + intermediate_tensors, + ) + + assert len(schedule_calls) == 1 + assert torch.equal(schedule_calls[0][1], expected_hidden_states) + assert schedule_calls[0][2] is expected_residual + assert schedule_calls[0][4] is afd_metadata + scheduled_hidden_states = expected_hidden_states + 1 + scheduled_residual = ( + torch.zeros_like(expected_hidden_states) + if expected_residual is None + else expected_residual + 2 + ) + if is_last_rank: + assert len(norm_calls) == 1 + assert torch.equal(output, scheduled_hidden_states + scheduled_residual) + else: + assert isinstance(output, FakeIntermediateTensors) + assert torch.equal(output["hidden_states"], scheduled_hidden_states) + assert torch.equal(output["residual"], scheduled_residual) + + def test_deepseek_afd_wrapper_keeps_full_model_compile_enabled(): source = Path("afd_plugin/model_executor/models/deepseek_v2.py").read_text() @@ -72,12 +278,16 @@ def test_deepseek_afd_attention_path_can_compute_gate_before_send(): "afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py", ).read_text() module_imports = source.split("logger = init_logger(__name__)", 1)[0] - forward_with_afd = source.split(" def forward_with_afd(", 1)[1].split( - " def forward_with_afd_v2(", + model_source = source.split("class AFDDeepseekV2Model", 1)[1].split( + "class AFDDeepseekV2ForCausalLM", + 1, + )[0] + model_forward = model_source.split(" def forward(", 1)[1].split( + " def compute_ffn_output(", 1, )[0] - forward_with_afd_v2 = source.split(" def forward_with_afd_v2(", 1)[1].split( - " def forward_with_afd_v3(", + gate_proxy = source.split("class GateOnlyRemoteMoE", 1)[1].split( + "class AFDDeepseekV2RemoteExpertsMoE", 1, )[0] attention_gate_forward = executor_source.split( @@ -85,18 +295,15 @@ def test_deepseek_afd_attention_path_can_compute_gate_before_send(): 1, )[1].split("def run_async_moe_ubatch_afd_forward(", 1)[0] - assert 'if self.afd_role == "attention":' in source + assert 'if afd_role == "attention":' in source assert "afd_plugin.model_executor.models.npu" not in module_imports - assert "from afd_plugin.model_executor.models.npu import (" in forward_with_afd_v2 - assert "deepseek_v2_async_cam_forward," in forward_with_afd_v2 assert "def _forward_attention(" not in source - assert "return self.forward_with_afd_v3(" in forward_with_afd - assert "return self.forward_with_afd_v2(" in forward_with_afd - assert ( - "return deepseek_v2_async_cam_forward.run_attention_gate_afd_forward(" - in forward_with_afd_v2 - ) - assert "layer.compute_attn_output(" not in forward_with_afd + assert "return super().forward(" in model_forward + assert "deepseek_v2_async_cam_forward.run_model_forward(" in model_forward + assert "compute_gate_topk(" in gate_proxy + assert "topk_weights=topk_weights" in gate_proxy + assert "topk_ids=topk_ids" in gate_proxy + assert "router_logits=router_logits" in gate_proxy assert "layer.compute_attn_output(" in attention_gate_forward assert "pending_ffn_recv" in attention_gate_forward assert "topk_weights" in attention_gate_forward @@ -138,18 +345,21 @@ def test_deepseek_afd_gate_on_attention_keeps_dense_layers_local(): "afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py", ).read_text() - assert "self.is_moe_layer = _is_moe_layer(config, layer_idx)" in source + assert "self.is_moe_layer = is_moe_layer" in source assert "self.compute_gate_on_attention and not self.is_moe_layer" in source assert "if not layer.is_moe_layer:" in executor_source - assert "self.is_dense_mlp_weight(name)" in source + assert ( + "return _ATTENTION_ROLE if compute_gate_on_attention else _FFN_ROLE" in source + ) -def test_deepseek_compute_gate_on_attention_is_npu_only(): +def test_deepseek_compute_gate_on_attention_selects_backend_boundary(): source = Path("afd_plugin/model_executor/models/deepseek_v2.py").read_text() - assert 'native.current_platform.device_type != "npu"' in source - assert "DeepSeekV2 compute_gate_on_attention is supported only on NPU" in source - assert "# NPU-only: non-NPU platforms are rejected before this branch." in source + assert 'device_type not in ("cuda", "npu")' in source + assert "self.mlp = AFDDeepseekV2RemoteExpertsMoE(" in source + assert "self.mlp = GateOnlyRemoteMoE(" in source + assert 'prefix=f"{prefix}.mlp"' in source assert ( "# NPU-only: Attention-side gate/topk is implemented in the NPU helper." in source @@ -161,12 +371,11 @@ def test_deepseek_compute_gate_on_attention_is_npu_only(): def test_deepseek_async_moe_ubatching_runs_attention_inside_stage_context(): - source = Path("afd_plugin/model_executor/models/deepseek_v2.py").read_text() executor_source = Path( "afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py", ).read_text() - forward_with_afd_v3 = source.split(" def forward_with_afd_v3(", 1)[1].split( - " def compute_ffn_output(", + model_forward = executor_source.split("def run_model_forward(", 1)[1].split( + "def run_attention_gate_afd_forward(", 1, )[0] async_ubatch_forward = executor_source.split( @@ -177,13 +386,8 @@ def test_deepseek_async_moe_ubatching_runs_attention_inside_stage_context(): 1, )[0] - assert "async_moe_ubatch_metadata" in forward_with_afd_v3 - assert ( - "return deepseek_v2_async_cam_forward.run_async_moe_ubatch_afd_forward(" - in forward_with_afd_v3 - ) - assert "from afd_plugin.model_executor.models.npu import (" in forward_with_afd_v3 - assert "deepseek_v2_async_cam_forward," in forward_with_afd_v3 + assert "async_moe_ubatch_metadata" in model_forward + assert "run_async_moe_ubatch_afd_forward(" in model_forward assert "_log_async_moe_forward_step(" not in async_ubatch_forward assert "first_moe_layer = int(model.config.first_k_dense_replace)" in ( async_ubatch_forward diff --git a/tests/unit/package/test_package.py b/tests/unit/package/test_package.py index 6966bc18..d5b171fe 100644 --- a/tests/unit/package/test_package.py +++ b/tests/unit/package/test_package.py @@ -105,6 +105,6 @@ def test_connectors_export_attn_output_without_recv_alias(): def test_vllm_version_support_is_exact_target(): - assert is_vllm_version_supported("0.19.1") - assert not is_vllm_version_supported("0.19.0") - assert not is_vllm_version_supported("0.19.2") + assert is_vllm_version_supported("0.26.0") + assert not is_vllm_version_supported("0.25.0") + assert not is_vllm_version_supported("0.26.1") diff --git a/tests/unit/v1/worker/test_attention_model_runner.py b/tests/unit/v1/worker/test_attention_model_runner.py index afb22642..82d06300 100644 --- a/tests/unit/v1/worker/test_attention_model_runner.py +++ b/tests/unit/v1/worker/test_attention_model_runner.py @@ -8,6 +8,7 @@ pytest.importorskip("torch") pytest.importorskip("vllm") +from vllm.config import CUDAGraphMode from vllm.v1.worker.gpu_model_runner import GPUModelRunner import afd_plugin.model_executor.models.forward_context as afd_forward_context @@ -580,22 +581,52 @@ def execute_model(_self, *args, **kwargs): execute_model, ) - result = runner.execute_model("scheduler", flag=True) + result = runner.execute_model("scheduler", "intermediate") assert runner.prof.steps == 1 - assert result == (("scheduler",), {"flag": True}) + assert result == (("scheduler", "intermediate"), {}) -def test_attention_runner_stops_gpu_profiler_on_shutdown(): +def test_attention_runner_preserves_native_shutdown(monkeypatch): runner = object.__new__(AFDAttentionModelRunner) runner.prof = _StepProfiler() runner.connector = _RecordingConnector() + native_shutdowns = [] + monkeypatch.setattr( + GPUModelRunner, + "shutdown", + lambda self: native_shutdowns.append(self), + ) + runner.shutdown() + assert native_shutdowns == [runner] assert runner.prof.stopped is True assert runner.connector.closed is True +def test_attention_warmup_preserves_profile_seq_lens(): + runner = object.__new__(AFDAttentionModelRunner) + runner.compilation_config = SimpleNamespace(cudagraph_num_of_warmups=1) + runner._is_warmup = False + runner._afd_pending_metadata = None + runner._afd_suppress_metadata_send = False + runner._afd_is_graph_capturing = False + runner._build_afd_metadata = lambda *_args: object() + runner._build_capture_dp_metadata = lambda *_args: object() + runner._send_dp_metadata = lambda *_args: None + dummy_runs = [] + runner._dummy_run = lambda *args, **kwargs: dummy_runs.append((args, kwargs)) + + runner._warmup_and_capture( + SimpleNamespace(num_tokens=8, uniform=True, num_active_loras=0), + CUDAGraphMode.FULL, + profile_seq_lens=7, + ) + + assert [kwargs["profile_seq_lens"] for _, kwargs in dummy_runs] == [7, 7] + + def test_forward_context_provider_installs_metadata_before_model_forward(monkeypatch): runner = object.__new__(AFDAttentionModelRunner) runner.afd_config = AFDConfig(role="attention") @@ -706,7 +737,7 @@ def test_attention_runner_builds_capture_dp_metadata_for_native_dp(): if hasattr(tokens, "tolist"): tokens = tokens.tolist() assert tokens == [64, 64] - assert int(metadata.max_tokens_across_dp_cpu) == 64 + assert not hasattr(metadata, "max_tokens_across_dp_cpu") def test_afd_rank_derives_from_data_parallel_rank(): diff --git a/tests/unit/v1/worker/test_dbo.py b/tests/unit/v1/worker/test_dbo.py index e3c24eaf..6c6b972f 100644 --- a/tests/unit/v1/worker/test_dbo.py +++ b/tests/unit/v1/worker/test_dbo.py @@ -16,7 +16,6 @@ def test_maybe_apply_dbo_yield_uses_custom_op(monkeypatch): calls = [] tensor = object() - yielded = object() monkeypatch.setattr( dbo, @@ -26,12 +25,40 @@ def test_maybe_apply_dbo_yield_uses_custom_op(monkeypatch): monkeypatch.setattr( dbo.torch.ops.vllm, "manual_dbo_yield", - lambda x: yielded if x is tensor else x, + lambda x: calls.append(("yield", x)), raising=False, ) - assert maybe_apply_dbo_yield(tensor, role="attention") is yielded - assert calls == ["register"] + assert maybe_apply_dbo_yield(tensor, role="attention") is tensor + assert calls == ["register", ("yield", tensor)] + + +def test_register_dbo_yield_custom_op_declares_input_mutation(monkeypatch): + registrations = [] + yield_calls = [] + tensor = object() + + monkeypatch.setattr(dbo, "_AFD_DBO_YIELD_OP_REGISTERED", False) + monkeypatch.setattr( + dbo, + "direct_register_custom_op", + lambda **kwargs: registrations.append(kwargs), + ) + monkeypatch.setattr( + dbo, + "_yield_if_dbo_enabled", + lambda: yield_calls.append("yield"), + ) + + dbo.register_dbo_yield_custom_op() + + assert len(registrations) == 1 + registration = registrations[0] + assert registration["op_name"] == "manual_dbo_yield" + assert registration["mutates_args"] == ["x"] + assert registration["op_func"](tensor) is None + assert yield_calls == ["yield"] + assert registration["fake_impl"](tensor) is None def test_maybe_apply_dbo_yield_does_not_probe_ascend(monkeypatch): diff --git a/tests/unit/v1/worker/test_ffn_model_runner.py b/tests/unit/v1/worker/test_ffn_model_runner.py index 4a128773..9f95207d 100644 --- a/tests/unit/v1/worker/test_ffn_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_model_runner.py @@ -7,27 +7,30 @@ import pytest -pytest.importorskip("torch") +torch = pytest.importorskip("torch") pytest.importorskip("vllm") -from afd_plugin.connectors import ( +import afd_plugin.v1.worker.ffn_model_runner as ffn_model_runner_module # noqa: E402 +from afd_plugin.connectors import ( # noqa: E402 AFDA2FTransferPayload, AFDControlPayload, + AFDExpertRoutingSpec, AFDTransferContext, AFDTransferMetadata, ) -from afd_plugin.v1.worker.cuda_graph import make_ffn_graph_key -from afd_plugin.v1.worker.ffn_model_runner import ( +from afd_plugin.v1.worker.cuda_graph import make_ffn_graph_key # noqa: E402 +from afd_plugin.v1.worker.ffn_model_runner import ( # noqa: E402 GPUFFNModelRunner, _set_moe_layer_index, ) -from afd_plugin.v1.worker.ffn_worker import AFDFFNWorker +from afd_plugin.v1.worker.ffn_worker import AFDFFNWorker # noqa: E402 class _FakeConnector: def __init__(self): self.attn_outputs = deque() self.ffn_outputs = [] + self.expert_routing_specs = [] self.dp_metadata_updates = [] self.closed = False # The runners reach the control plane through connector.control_plane; @@ -44,7 +47,9 @@ def update_state_from_dp_metadata(self, payload): ), ) - def recv_attn_output(self, ubatch_idx=None): + def recv_attn_output(self, ubatch_idx=None, routing_spec=None): + if routing_spec is not None: + self.expert_routing_specs.append(routing_spec) if ubatch_idx is None: return self.attn_outputs.popleft() for item in tuple(self.attn_outputs): @@ -67,6 +72,9 @@ def __init__(self): class _FakeModel: + def get_experts_layer_indices(self): + return () + def compute_ffn_output(self, hidden_states, layer_idx): return f"ffn({hidden_states}, layer={layer_idx})" @@ -112,6 +120,7 @@ def _runner_with_connector_and_model(model, *, num_layers=1): parallel_config=SimpleNamespace( data_parallel_size=1, is_moe_model=True, + use_sequence_parallel_moe=False, ), compilation_config=SimpleNamespace( fast_moe_cold_start=False, @@ -120,6 +129,8 @@ def _runner_with_connector_and_model(model, *, num_layers=1): ) runner.connector = _FakeConnector() runner.model = model + runner.afd_config = SimpleNamespace(compute_gate_on_attention=False) + runner.afd_cudagraph_policy = SimpleNamespace(enabled=False) runner.num_layers = num_layers runner.use_cuda_graph = False runner._cuda_graphs = {} @@ -142,10 +153,14 @@ def _tokens(dp_metadata): class _FakeGraph: def __init__(self): self.replay_count = 0 + self.reset_count = 0 def replay(self): self.replay_count += 1 + def reset(self): + self.reset_count += 1 + def test_ffn_runner_executes_model_compute_ffn_output(): runner = _runner_with_connector_and_model(_FakeModel()) @@ -168,14 +183,13 @@ def test_ffn_runner_executes_model_compute_ffn_output(): assert metadata.layer_idx == 0 -def test_ffn_runner_passthrough_without_model_compute_hook(): +def test_ffn_runner_exposes_missing_model_contract(): runner = _runner_with_connector_and_model(SimpleNamespace()) metadata = _metadata() runner.connector.attn_outputs.append(_payload("hidden", metadata)) - runner.execute_model(dp_metadata_list={0: _FakeDPMetadata([1])}) - - assert runner.connector.ffn_outputs == [("hidden", metadata)] + with pytest.raises(AttributeError, match="get_experts_layer_indices"): + runner.execute_model(dp_metadata_list={0: _FakeDPMetadata([1])}) def test_ffn_runner_processes_each_ubatch_for_each_layer(): @@ -208,6 +222,225 @@ def test_ffn_runner_processes_each_ubatch_for_each_layer(): ] +def test_ffn_side_gate_mixes_dense_and_experts_protocols(): + class _MixedModel(_FakeModel): + def __init__(self): + self.calls = [] + + def get_experts_layer_indices(self): + return (1,) + + def compute_ffn_output(self, hidden_states, layer_idx): + self.calls.append((hidden_states, layer_idx)) + return f"ffn({hidden_states}, layer={layer_idx})" + + model = _MixedModel() + runner = _runner_with_connector_and_model(model, num_layers=2) + dense_metadata = _metadata() + expert_context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=1, + stage_idx=0, + seq_len=1, + ), + ) + runner.connector.attn_outputs.append(_payload("dense-hidden", dense_metadata)) + runner.connector.attn_outputs.append( + AFDA2FTransferPayload( + hidden_states="moe-hidden", + context=expert_context, + ), + ) + + runner.execute_model(dp_metadata_list={0: _FakeDPMetadata([1])}) + + assert model.calls == [("dense-hidden", 0), ("moe-hidden", 1)] + assert runner.connector.ffn_outputs == [ + ("ffn(dense-hidden, layer=0)", dense_metadata), + ("ffn(moe-hidden, layer=1)", expert_context.metadata), + ] + + +def test_attention_side_gate_processes_only_experts_layers(): + router_logits = object() + + class _AttentionGateModel(_FakeModel): + def __init__(self): + self.calls = [] + + def get_experts_layer_indices(self): + return (1,) + + def get_experts_routing_spec(self, layer_idx): + return AFDExpertRoutingSpec( + router_logits_width=4, + router_logits_dtype=torch.float32, + ) + + def compute_experts_output( + self, + hidden_states, + layer_idx, + received_router_logits, + ): + self.calls.append( + (hidden_states, layer_idx, received_router_logits), + ) + return "expert-output" + + model = _AttentionGateModel() + runner = _runner_with_connector_and_model(model, num_layers=2) + runner.afd_config.compute_gate_on_attention = True + expert_context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=1, + stage_idx=0, + seq_len=1, + ), + ) + runner.connector.attn_outputs.append( + AFDA2FTransferPayload( + hidden_states="moe-hidden", + context=expert_context, + router_logits=router_logits, + ), + ) + + runner.execute_model(dp_metadata_list={0: _FakeDPMetadata([1])}) + + assert model.calls == [("moe-hidden", 1, router_logits)] + assert runner.connector.ffn_outputs == [ + ("expert-output", expert_context.metadata), + ] + + +def test_experts_graph_capture_passes_model_routing_spec(): + routing_spec = AFDExpertRoutingSpec( + router_logits_width=4, + router_logits_dtype=torch.float32, + ) + router_logits = object() + + class _GraphModel(_FakeModel): + def get_experts_layer_indices(self): + return (1,) + + def get_experts_routing_spec(self, layer_idx): + assert layer_idx == 1 + return routing_spec + + def compute_experts_output( + self, + hidden_states, + layer_idx, + received_router_logits, + ): + assert (hidden_states, layer_idx, received_router_logits) == ( + "moe-hidden", + 1, + router_logits, + ) + return "expert-output" + + runner = _runner_with_connector_and_model(_GraphModel(), num_layers=2) + runner.afd_config.compute_gate_on_attention = True + runner.afd_cudagraph_policy.enabled = True + expert_context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=1, + stage_idx=0, + seq_len=1, + ), + ) + runner.connector.attn_outputs.append( + AFDA2FTransferPayload( + hidden_states="moe-hidden", + context=expert_context, + router_logits=router_logits, + ), + ) + + runner._ffn_forward( + dp_metadata_list={0: _FakeDPMetadata([1])}, + is_graph_capturing=True, + ) + + assert runner.connector.expert_routing_specs == [routing_spec] + assert runner.connector.ffn_outputs == [ + ("expert-output", expert_context.metadata), + ] + + +@pytest.mark.parametrize( + "is_warmup", + [False, True], + ids=["compiled-policy", "warmup"], +) +def test_experts_pass_static_routing_spec_for_each_stage(is_warmup): + routing_spec = AFDExpertRoutingSpec( + router_logits_width=4, + router_logits_dtype=torch.float32, + ) + + class _ExpertsModel(_FakeModel): + def get_experts_layer_indices(self): + return (1,) + + def get_experts_routing_spec(self, layer_idx): + assert layer_idx == 1 + return routing_spec + + def compute_experts_output( + self, + hidden_states, + layer_idx, + received_router_logits, + ): + return f"experts({hidden_states}, {layer_idx}, {received_router_logits})" + + runner = _runner_with_connector_and_model(_ExpertsModel(), num_layers=2) + runner.afd_config.compute_gate_on_attention = True + runner.afd_cudagraph_policy.enabled = True + contexts = [] + for stage_idx in (1, 0): + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=1, + stage_idx=stage_idx, + seq_len=1, + ), + ) + contexts.append(context) + runner.connector.attn_outputs.append( + AFDA2FTransferPayload( + hidden_states=f"hidden-{stage_idx}", + context=context, + router_logits=f"router-{stage_idx}", + ), + ) + + runner.execute_model( + dp_metadata_list={ + 0: _FakeDPMetadata([1]), + 1: _FakeDPMetadata([1]), + }, + is_warmup=is_warmup, + ) + + assert len(runner.connector.dp_metadata_updates) == 1 + dp_metadata_update, is_graph_capturing, reported_is_warmup = ( + runner.connector.dp_metadata_updates[0] + ) + assert sorted(dp_metadata_update) == [0, 1] + assert is_graph_capturing is False + assert reported_is_warmup is is_warmup + assert runner.connector.expert_routing_specs == [routing_spec, routing_spec] + assert runner.connector.ffn_outputs == [ + ("experts(hidden-0, 1, router-0)", contexts[1].metadata), + ("experts(hidden-1, 1, router-1)", contexts[0].metadata), + ] + + def test_ffn_runner_requires_dp_metadata_list(): runner = object.__new__(GPUFFNModelRunner) runner.prof = None @@ -265,12 +498,31 @@ def test_ffn_runner_steps_gpu_profiler(): assert runner.prof.steps == 1 -def test_ffn_runner_stops_gpu_profiler_on_shutdown(): +def test_ffn_runner_releases_owned_runtime_state_on_shutdown(monkeypatch): runner = _runner_with_connector_and_model(_FakeModel()) runner.prof = _StepProfiler() + graph = _FakeGraph() + runner._cuda_graphs = {("graph",): {"graph": graph}} + runner._graph_memory_pool = object() + runner.vllm_config.compilation_config.static_forward_context["layer"] = object() + rope_cache = {"rope": object()} + workspace_resets = [] + monkeypatch.setattr(ffn_model_runner_module, "_ROPE_DICT", rope_cache) + monkeypatch.setattr( + ffn_model_runner_module, + "reset_workspace_manager", + lambda: workspace_resets.append(True), + ) runner.shutdown() + assert graph.reset_count == 1 + assert runner._cuda_graphs == {} + assert runner._graph_memory_pool is None + assert runner.vllm_config.compilation_config.static_forward_context == {} + assert runner.model is None + assert rope_cache == {} + assert workspace_resets == [True] assert runner.prof.stopped is True assert runner.connector.closed is True @@ -314,6 +566,15 @@ def test_ffn_worker_scheduler_execute_model_fails_fast(): worker.execute_model(scheduler_output=object()) +def test_ffn_worker_reports_zero_compilation_times(): + worker = object.__new__(AFDFFNWorker) + + compilation_times = worker.compile_or_warm_up_model() + + assert compilation_times.language_model == 0.0 + assert compilation_times.encoder == 0.0 + + def test_ffn_worker_loop_rejects_connector_without_control_plane(): worker = object.__new__(AFDFFNWorker) event = threading.Event() diff --git a/tests/unit/v1/worker/test_runtime_classpaths.py b/tests/unit/v1/worker/test_runtime_classpaths.py index 081b8c9b..36343950 100644 --- a/tests/unit/v1/worker/test_runtime_classpaths.py +++ b/tests/unit/v1/worker/test_runtime_classpaths.py @@ -1,5 +1,7 @@ from __future__ import annotations +import inspect + import pytest from afd_plugin.validation import ( @@ -31,6 +33,86 @@ NPU_FFN_MODEL_RUNNER_FQCN, ] +V026_OVERRIDE_CONTRACTS = [ + ("attention_worker", "AFDAttentionWorker", "Worker", "__init__"), + ("attention_worker", "AFDAttentionWorker", "Worker", "init_device"), + ("ffn_worker", "AFDFFNWorker", "Worker", "__init__"), + ("ffn_worker", "AFDFFNWorker", "Worker", "init_device"), + ("ffn_worker", "AFDFFNWorker", "Worker", "get_kv_cache_spec"), + ("ffn_worker", "AFDFFNWorker", "Worker", "initialize_from_config"), + ("ffn_worker", "AFDFFNWorker", "Worker", "compile_or_warm_up_model"), + ("ffn_worker", "AFDFFNWorker", "Worker", "execute_model"), + ("ffn_worker", "AFDFFNWorker", "Worker", "shutdown"), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "load_model", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "_build_attention_metadata", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "_determine_batch_execution_and_padding", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "_model_forward", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "execute_model", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "_dummy_run", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "_warmup_and_capture", + ), + ( + "attention_model_runner", + "AFDAttentionModelRunner", + "GPUModelRunner", + "shutdown", + ), + ("ubatch_wrapper", "AFDUBatchWrapper", "UBatchWrapper", "__init__"), + ( + "ubatch_wrapper", + "AFDUBatchWrapper", + "UBatchWrapper", + "_create_sm_control_context", + ), + ( + "ubatch_wrapper", + "AFDUBatchWrapper", + "UBatchWrapper", + "_make_ubatch_metadata", + ), +] + + +def _call_contract(callable_obj): + return [ + (parameter.name, parameter.kind, parameter.default) + for parameter in inspect.signature(callable_obj).parameters.values() + ] + @pytest.mark.parametrize( "qualname", @@ -52,6 +134,43 @@ def test_gpu_runtime_class_paths_resolve_when_vllm_is_available(qualname): assert cls.__module__.startswith("afd_plugin.v1.worker") +@pytest.mark.vllm_runtime +@pytest.mark.parametrize( + ("module_name", "afd_class_name", "native_class_name", "method_name"), + V026_OVERRIDE_CONTRACTS, +) +def test_gpu_v1_overrides_match_native_call_contract( + module_name, + afd_class_name, + native_class_name, + method_name, +): + pytest.importorskip("torch") + pytest.importorskip("vllm") + + if module_name == "attention_worker": + from vllm.v1.worker import gpu_worker as native_module + + from afd_plugin.v1.worker import attention_worker as afd_module + elif module_name == "ffn_worker": + from vllm.v1.worker import gpu_worker as native_module + + from afd_plugin.v1.worker import ffn_worker as afd_module + elif module_name == "attention_model_runner": + from vllm.v1.worker import gpu_model_runner as native_module + + from afd_plugin.v1.worker import attention_model_runner as afd_module + else: + from vllm.v1.worker import gpu_ubatch_wrapper as native_module + + from afd_plugin.v1.worker import ubatch_wrapper as afd_module + + afd_method = getattr(getattr(afd_module, afd_class_name), method_name) + native_method = getattr(getattr(native_module, native_class_name), method_name) + + assert _call_contract(afd_method) == _call_contract(native_method) + + @pytest.mark.vllm_runtime @pytest.mark.parametrize("qualname", NPU_RUNTIME_CLASS_PATHS) def test_npu_runtime_class_paths_resolve_when_vllm_ascend_is_available(qualname): diff --git a/uv.lock b/uv.lock index f07ce359..042960ef 100644 --- a/uv.lock +++ b/uv.lock @@ -2,10 +2,14 @@ version = 1 revision = 3 requires-python = ">=3.10, <3.14" resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", ] [[package]] @@ -169,31 +173,31 @@ wheels = [ [[package]] name = "apache-tvm-ffi" -version = "0.1.11" +version = "0.1.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/4b9226cd45aa800a6904603dda9b323d728f3c3869952a673f3483b78b19/apache_tvm_ffi-0.1.11.tar.gz", hash = "sha256:153cd2c5a9717804cb0bcd9b2709f22a1e5f80ed05b5a490faf5949b136eedba", size = 2798354, upload-time = "2026-05-04T17:48:43.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/07/c17d9608f4c9e4637091210a07e30be0f34b303ecb46bd5a0dd2838a2e0d/apache_tvm_ffi-0.1.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3587eb393096832d356be94b2241c6f13b8f41ff729556ce0dc69a4fc7fed73a", size = 2464945, upload-time = "2026-05-04T17:47:33.715Z" }, - { url = "https://files.pythonhosted.org/packages/e9/91/c9227ef7d42ecd8f48b0ba833d704178325147dfce1d95dca96787eef0bd/apache_tvm_ffi-0.1.11-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e7527775369a32964e083fed04a3a1ce4134e3e8719a64660ad976be2d0ee58e", size = 2675161, upload-time = "2026-05-04T17:47:36.101Z" }, - { url = "https://files.pythonhosted.org/packages/23/29/87d7931157e47c1a2d5cdb13f306d2c20d064d4e0ba0abd34f6a4b3d8b34/apache_tvm_ffi-0.1.11-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c5b2b6ae779008ab1586866dc04ab8d5798e5cf6c240df675b78501c6c6f8c95", size = 2789472, upload-time = "2026-05-04T17:47:38.328Z" }, - { url = "https://files.pythonhosted.org/packages/c5/36/0f3cb67ccfb2fa6ab57b14797c4fe524770a5319976cca2a3c185fda2cd1/apache_tvm_ffi-0.1.11-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66d9a23689070d8c3d3c3a47b3f2624f7b160ab155b6b8283d9b16b1a94e50d1", size = 2585760, upload-time = "2026-05-04T17:47:40.336Z" }, - { url = "https://files.pythonhosted.org/packages/3e/85/934780944c78d3a3c9696f84f1b80d14843a4220a8cbcc90ce0f25e289fb/apache_tvm_ffi-0.1.11-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a565f6bf25adf588576578d7e2a272b09afef4084c4b668807045bd9e1ee89a9", size = 2768000, upload-time = "2026-05-04T17:47:42.428Z" }, - { url = "https://files.pythonhosted.org/packages/5c/cc/0f21e0eea204a19cd251a6cc24fe494d07ecdefc11b35be2f8c0e110a1d5/apache_tvm_ffi-0.1.11-cp310-cp310-win_amd64.whl", hash = "sha256:119849c342bd97a9d76ec58eb77dc8dff4ebfbe8b17ea72280c5e5b103277ebe", size = 2407386, upload-time = "2026-05-04T17:47:44.492Z" }, - { url = "https://files.pythonhosted.org/packages/67/df/e573d324e3c7cf77fb526d26d59ec0c365e858f5e09f0742dbc39878a100/apache_tvm_ffi-0.1.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5632f5b4d3af46cb6ccc846120418ad478174e896589ba040bc5a4e7a7356716", size = 2462827, upload-time = "2026-05-04T17:47:46.552Z" }, - { url = "https://files.pythonhosted.org/packages/8f/22/aec1d70baa4bc1e3962a23439f82099f7775992cc5b70a19d4a8ef2a47e4/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2f0d4b165f371d2dee6013e47353d178b01742171fd1092c654cbbc0fa5c6d60", size = 2675459, upload-time = "2026-05-04T17:47:48.63Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b6/84acc663a43ba6e72b4dfd8d923fb5a2d1eb2c867b5b2067c18cb3dc855a/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2cf501753d7693daa73711a27f0f9d9f0f76e9e7d98f2fc2403f423ee7bbfd9b", size = 2789376, upload-time = "2026-05-04T17:47:50.449Z" }, - { url = "https://files.pythonhosted.org/packages/ac/96/e216d5d0f420ccf54775c913b6506755f65bc262511a9948b4d1387bcbc9/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a051c84985be3f9d8a20a16ec4bdba73a7ae01d3fb2f18a2c72bbd7a28aaa155", size = 2584233, upload-time = "2026-05-04T17:47:52.387Z" }, - { url = "https://files.pythonhosted.org/packages/72/9c/af12a5e796a672664f2f18eca989222697b91974a9c9e98d4ec2ecfeeb83/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87f84e7c2393fadac340fd179a631a697effe54d7317b2543e0930452a0a673d", size = 2768221, upload-time = "2026-05-04T17:47:54.057Z" }, - { url = "https://files.pythonhosted.org/packages/3e/f2/7d1c3c13f1cfd479144a41ebcc5b206337b5b02949d0348c739c7fca2079/apache_tvm_ffi-0.1.11-cp311-cp311-win_amd64.whl", hash = "sha256:fd587ecd8ee843bbec467762490c8347af3dfe997608f9841b48a98f5fffac7f", size = 2409048, upload-time = "2026-05-04T17:47:55.824Z" }, - { url = "https://files.pythonhosted.org/packages/05/9d/0f81ca556e5836b3ca64818cdae3f47dc7822bd35d22ddef7a54106d801d/apache_tvm_ffi-0.1.11-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ae51cc7df415b5f373a9df4baa1165a65608e519bea81e7dd23428f00eeb689", size = 2418793, upload-time = "2026-05-04T17:47:57.879Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a9/f48e5dd4ae1f6f0c5ffac259c0a9531b7d6a7c0a4c45bc2229d55de6adf8/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da2c8d07fdc737d1ba75f4de25c29f156905b9dc980f1da90c395b4db525f522", size = 2605176, upload-time = "2026-05-04T17:47:59.676Z" }, - { url = "https://files.pythonhosted.org/packages/36/99/2848df4e8ed5bf51df1d286d1718510584fa61e88adbc9c5b23d71b38f7c/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78aa1857b04a2ea718317041ab3f01288b3d496e6036eb1b99ebdc9da0fdaef5", size = 2725887, upload-time = "2026-05-04T17:48:01.381Z" }, - { url = "https://files.pythonhosted.org/packages/7d/80/963c991934a4eb0fa0c0178f51963333fe14a96b732009da642b6bf6b42e/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8b845c8dff498fb981c1dda36c954549204191b485a385845e604966594d0b2", size = 2513121, upload-time = "2026-05-04T17:48:03.43Z" }, - { url = "https://files.pythonhosted.org/packages/4d/18/95569107ee83619d61a3bb0d28743a0599f85c5161981e3e098c82c2b185/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2843f084cdc94dedacd8b257a395a2b71b8a3dc7fc99711b148bf1d161983128", size = 2697683, upload-time = "2026-05-04T17:48:05.222Z" }, - { url = "https://files.pythonhosted.org/packages/dc/99/f352cf1cce8f6f05584c4adf11de9eca07e6d217229bad6af35fb372926c/apache_tvm_ffi-0.1.11-cp312-abi3-win_amd64.whl", hash = "sha256:bd67e03759d25ff59f4e0ed9c8630a16872afc9dd8792f46ac3c927554015e60", size = 2365545, upload-time = "2026-05-04T17:48:07.295Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/17/b0/5114e30faffe3279a51a5f3b45dd1b7ce09af1246b62447b45a39a374e54/apache_tvm_ffi-0.1.10.tar.gz", hash = "sha256:974c208766c304c780c17c6d405449e862f83b22c7b6b2b8c28b29d55a806ae3", size = 2691605, upload-time = "2026-04-07T19:58:51.767Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/3b/8c850c36a522e8b3d57bd209b94464c98066cf9c0550a1c8af708b09669b/apache_tvm_ffi-0.1.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ef1059d8f8ae6e497440b94805b30f9ff5db21cd0b1745c8520b2874d3eb9efb", size = 2331240, upload-time = "2026-04-07T19:57:53.837Z" }, + { url = "https://files.pythonhosted.org/packages/e1/09/61c294a0b72b37071e5227838a2ee56681d4bfe154b387eb6fbbb8f1d073/apache_tvm_ffi-0.1.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b4a3381f6e93f675217bf5421bd21a1ee1f3841c522a588d42dc37d9c9148108", size = 2544126, upload-time = "2026-04-07T19:57:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/d4444fb595c5d8f9309a5587f961d28a2918d02cf88d386a36d788ef8085/apache_tvm_ffi-0.1.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28dbe0c1d8c5c43b7d3574d1c9f0606437e7008efbd2b7cb118385465815e45d", size = 2651634, upload-time = "2026-04-07T19:57:57.411Z" }, + { url = "https://files.pythonhosted.org/packages/24/e2/03f8af49c08aabaad292296523280ee2b1c10982baf6411aea75fe3a01cb/apache_tvm_ffi-0.1.10-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dec05560e84b007998795484a3306965fe1d8de7d8de7e8c0bb7ccd84a246336", size = 2461544, upload-time = "2026-04-07T19:57:59.305Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e3/436b573a23ef171c14e90181ff379819ef8481d8236fda9afb29b3b15516/apache_tvm_ffi-0.1.10-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5dda14f065520c3bdec6889fecfa7c1b06a4f6fb23e7b2475d9a0477eb8588d8", size = 2632276, upload-time = "2026-04-07T19:58:00.962Z" }, + { url = "https://files.pythonhosted.org/packages/29/a8/d1a17ac7d85183bfceaa26efa5bda9093010d00c9da70fd852baf3c37224/apache_tvm_ffi-0.1.10-cp310-cp310-win_amd64.whl", hash = "sha256:d9109b81b2584a1a2f8bf40bc92f2a187ea848573796c210c13379535a0404f7", size = 2303306, upload-time = "2026-04-07T19:58:02.667Z" }, + { url = "https://files.pythonhosted.org/packages/54/1b/05b0581b9d4ebb406f717533ec1f984ae3e020c15da37518ee1ac663f2da/apache_tvm_ffi-0.1.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e6fb3b33e0ab087de3a0fa3803dbd48a9acbaddee61bd2cc13bd8ad7ea87d0e7", size = 2329920, upload-time = "2026-04-07T19:58:04.017Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/598da8bf49e850aa329a024929643eb141d7907f4d97705b74e49ca499f6/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5cf055a83e1b1944dd05386c593bc22de29a1aeb6cae45af54735796875194a", size = 2543849, upload-time = "2026-04-07T19:58:05.419Z" }, + { url = "https://files.pythonhosted.org/packages/50/58/221b41c5f77405f99875754f2a38c01da49387e366bf0fd40302b2cd25f3/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81c4144fc06750312f2829960862bd52ba6f0bb17e6d7aae3f7a09f9170f7e7a", size = 2650260, upload-time = "2026-04-07T19:58:07.002Z" }, + { url = "https://files.pythonhosted.org/packages/01/2b/36b5210d24492dc4dda488d785dd4039c0788238f6aa4aa5067b2ea494d1/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7bafe9a6191c77f3978e9cd9726799abbe7fd574913fa2416402bc876633524e", size = 2459987, upload-time = "2026-04-07T19:58:08.409Z" }, + { url = "https://files.pythonhosted.org/packages/9f/36/8f8f719c1c52ed978fc99acde51827f5fc48380e69a310a02a6a5ae94d0f/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2ba653825f806a87fe2ca48ebab1abb9ae0f17d6642fbada622c6c5eea9fe96", size = 2631364, upload-time = "2026-04-07T19:58:09.784Z" }, + { url = "https://files.pythonhosted.org/packages/65/64/4ec0ea8eebc79b17dd8bdcf06c809b5ae5ff58aa9c3ffbe8dd26b976d55f/apache_tvm_ffi-0.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:8009ec2a9ca5c04cd8686102f2d3b648dfa5a3cb2ceb57a21f03f7b8480a58fb", size = 2304477, upload-time = "2026-04-07T19:58:11.183Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/0ba672dba52f9ecc813ce7ff4ef4aa5a2c5f27243d26165f09053f057a76/apache_tvm_ffi-0.1.10-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:52ed8fec82451c3af1e205f55500e5adc5eaa1913c82ce15b2064d305d7f880b", size = 2285850, upload-time = "2026-04-07T19:58:12.784Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2a/1978a1c827e1212de4f369ec08cfeb44719bbe6cbeab90b15e967c68c108/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ec5c4a81e294e6379e4dea68c86266924d3f22829c3de272806c980238e43e59", size = 2476596, upload-time = "2026-04-07T19:58:14.316Z" }, + { url = "https://files.pythonhosted.org/packages/50/6f/23740f06829030704e6f8f1f7093a06b7a68f904baa40053a5f594705bae/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:73d478395a8625dd92fde7b7fd92b4719f18f480b78336e422cb66cc7985213d", size = 2589574, upload-time = "2026-04-07T19:58:15.94Z" }, + { url = "https://files.pythonhosted.org/packages/92/d0/54badf5c8f6208e06f331a20ddd154f19c94c2e906da5b8cce7d60727d4b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3829216a8500c2f61062e48c627f6db6c3fa49416b3ffa85bc04243ae5d759f7", size = 2396434, upload-time = "2026-04-07T19:58:17.519Z" }, + { url = "https://files.pythonhosted.org/packages/51/f7/ca3fdadc2468e8b67a2f3f13bb7aa132c584feefd8a25dbf920e4bf0a03b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96b69030c722572e13e30182733adfa2d604258e988b3f6630a16f397c7f9288", size = 2571084, upload-time = "2026-04-07T19:58:20.399Z" }, + { url = "https://files.pythonhosted.org/packages/23/2d/bf899e1ba4ea1da6a55a04ad3e9c07338ee06a140862b05310bae9a00cf9/apache_tvm_ffi-0.1.10-cp312-abi3-win_amd64.whl", hash = "sha256:14e59f6f69881d37a25b03943cfac33317a06f6745df0ff2dfb3b0cd3ed3698f", size = 2261853, upload-time = "2026-04-07T19:58:21.772Z" }, ] [[package]] @@ -223,6 +227,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "backports-strenum" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/c7/2ed54c32fed313591ffb21edbd48db71e68827d43a61938e5a0bc2b6ec91/backports_strenum-1.3.1.tar.gz", hash = "sha256:77c52407342898497714f0596e86188bb7084f89063226f4ba66863482f42414", size = 7257, upload-time = "2023-12-09T14:36:40.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/50/56cf20e2ee5127b603b81d5a69580a1a325083e2b921aa8f067da83927c0/backports_strenum-1.3.1-py3-none-any.whl", hash = "sha256:cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83", size = 8304, upload-time = "2023-12-09T14:36:39.905Z" }, +] + [[package]] name = "blake3" version = "1.0.8" @@ -526,7 +539,7 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.15.0.1" +version = "0.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "loguru" }, @@ -534,9 +547,9 @@ dependencies = [ { name = "torch" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/41/1b/c3c4a98ec5f2727656336f07a0c35862195c310d8eb0b2fa5b4be6848680/compressed_tensors-0.15.0.1.tar.gz", hash = "sha256:a8e93054e8a5ec49c980b09ed36c4c1249b4a8ee167920a8e461c4da26e78d99", size = 229412, upload-time = "2026-04-10T14:23:54.708Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/9e/d7f18bd9a0354088abc11a0c1f2c7698f7c49e5a709faedf6a46e388f693/compressed_tensors-0.17.0.tar.gz", hash = "sha256:15c20d06bdbcf35b51fc99fd125e7b9be1e1855567c33b7a46dfac26ad6fb126", size = 257091, upload-time = "2026-06-03T16:49:17.208Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/52/93833dc1610e017ac5b7dcd59b8304d8ef67d1114c2d124e728a2cbbea12/compressed_tensors-0.15.0.1-py3-none-any.whl", hash = "sha256:e1b1f322e82e475715e242bad46925a304ea8e5c98b5055a15b8eb22fb6bfea9", size = 194260, upload-time = "2026-04-10T14:23:53.098Z" }, + { url = "https://files.pythonhosted.org/packages/35/63/6edf0415b072fff0bf8b546074dea3f0f9b148e49b601ac98bdc60a76c68/compressed_tensors-0.17.0-py3-none-any.whl", hash = "sha256:4a1b89b508f7efb8ffb4eee8a6e69e0452d9b080cae130146025c64fbe9fa9aa", size = 211714, upload-time = "2026-06-03T16:49:15.672Z" }, ] [[package]] @@ -589,25 +602,66 @@ wheels = [ name = "cuda-bindings" version = "12.9.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "cuda-pathfinder", marker = "sys_platform == 'darwin'" }, +] + +[[package]] +name = "cuda-bindings" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", +] +dependencies = [ + { name = "cuda-pathfinder", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, + { url = "https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0", size = 6675244, upload-time = "2026-05-29T23:11:38.664Z" }, + { url = "https://files.pythonhosted.org/packages/fc/64/bb17e4d168569ef7be05c44474fe3dc19278d60a69ba228e45a431c86444/cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051", size = 5625597, upload-time = "2026-05-29T23:11:40.808Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, + { url = "https://files.pythonhosted.org/packages/95/7a/c5e3c34a409b148f5c0f5a4ea374158f95d488862c1dffedf9aa5c639df9/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708", size = 6674166, upload-time = "2026-05-29T23:11:45.478Z" }, + { url = "https://files.pythonhosted.org/packages/93/f7/0e35987a21914f84068061dcf4b61466ccbce1c62ddc9727596d5ed0c26f/cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1", size = 5664286, upload-time = "2026-05-29T23:11:47.719Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/7c/95/872a0392122f1fb43fcb06869790ef3171f37beee9f7db8f441739113570/cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff", size = 5875099, upload-time = "2026-05-29T23:11:54.635Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, + { url = "https://files.pythonhosted.org/packages/0c/2f/6a0dd496550c6fafbf6aeb1bf40242eeabb2fd138a43892aabb4be8224c2/cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202", size = 5830027, upload-time = "2026-05-29T23:12:01.205Z" }, +] + +[[package]] +name = "cuda-core" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "backports-strenum", marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'darwin'" }, + { name = "numpy", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/37/31/bfcc870f69c6a017c4ad5c42316207fc7551940db6f3639aa4466ec5faf3/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a022c96b8bd847e8dc0675523431149a4c3e872f440e3002213dbb9e08f0331a", size = 11800959, upload-time = "2025-10-21T14:51:26.458Z" }, - { url = "https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5", size = 12148218, upload-time = "2025-10-21T14:51:28.855Z" }, - { url = "https://files.pythonhosted.org/packages/b5/1e/9c8ed3f3dbed7b7d038805fdc65cbc65fda9983e84437778a9571e7092bc/cuda_bindings-12.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:f69107389e6b9948969bfd0a20c4f571fd1aefcfb1d2e1b72cc8ba5ecb7918ab", size = 11464568, upload-time = "2025-10-21T14:51:31.454Z" }, - { url = "https://files.pythonhosted.org/packages/a9/2b/ebcbb60aa6dba830474cd360c42e10282f7a343c0a1f58d24fbd3b7c2d77/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6a429dc6c13148ff1e27c44f40a3dd23203823e637b87fd0854205195988306", size = 11840604, upload-time = "2025-10-21T14:51:34.565Z" }, - { url = "https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c912a3d9e6b6651853eed8eed96d6800d69c08e94052c292fec3f282c5a817c9", size = 12210593, upload-time = "2025-10-21T14:51:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/dd/be/90d32049e06abcfba4b2e7df1dbcb5e16215c8852eef0cd8b25f38a66bd4/cuda_bindings-12.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:443b0875916879c2e4c3722941e25e42d5ab9bcbf34c9e83404fb100fa1f6913", size = 11490933, upload-time = "2025-10-21T14:51:38.792Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c2/65bfd79292b8ff18be4dd7f7442cea37bcbc1a228c1886f1dea515c45b67/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:694ba35023846625ef471257e6b5a4bc8af690f961d197d77d34b1d1db393f56", size = 11760260, upload-time = "2025-10-21T14:51:40.79Z" }, - { url = "https://files.pythonhosted.org/packages/a9/c1/dabe88f52c3e3760d861401bb994df08f672ec893b8f7592dc91626adcf3/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fda147a344e8eaeca0c6ff113d2851ffca8f7dfc0a6c932374ee5c47caa649c8", size = 12151019, upload-time = "2025-10-21T14:51:43.167Z" }, - { url = "https://files.pythonhosted.org/packages/df/6b/9c1b1a6c01392bfdd758e9486f52a1a72bc8f49e98f9355774ef98b5fb4e/cuda_bindings-12.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:696ca75d249ddf287d01b9a698b8e2d8a05046495a9c051ca15659dc52d17615", size = 11586961, upload-time = "2025-10-21T14:51:45.394Z" }, - { url = "https://files.pythonhosted.org/packages/05/8b/b4b2d1c7775fa403b64333e720cfcfccef8dcb9cdeb99947061ca5a77628/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf8bfaedc238f3b115d957d1fd6562b7e8435ba57f6d0e2f87d0e7149ccb2da5", size = 11570071, upload-time = "2025-10-21T14:51:47.472Z" }, - { url = "https://files.pythonhosted.org/packages/63/56/e465c31dc9111be3441a9ba7df1941fe98f4aa6e71e8788a3fb4534ce24d/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:32bdc5a76906be4c61eb98f546a6786c5773a881f3b166486449b5d141e4a39f", size = 11906628, upload-time = "2025-10-21T14:51:49.905Z" }, - { url = "https://files.pythonhosted.org/packages/05/d0/d0e4e2e047d8e899f023fa15ad5e9894ce951253f4c894f1cd68490fdb14/cuda_bindings-12.9.4-cp313-cp313-win_amd64.whl", hash = "sha256:a2e82c8985948f953c2be51df45c3fe11c812a928fca525154fb9503190b3e64", size = 11556719, upload-time = "2025-10-21T14:51:52.248Z" }, - { url = "https://files.pythonhosted.org/packages/ec/07/6aff13bc1e977e35aaa6b22f52b172e2890c608c6db22438cf7ed2bf43a6/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3adf4958dcf68ae7801a59b73fb00a8b37f8d0595060d66ceae111b1002de38d", size = 11566797, upload-time = "2025-10-21T14:51:54.581Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/1e6be415e37478070aeeee5884c2022713c1ecc735e6d82d744de0252eee/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56e0043c457a99ac473ddc926fe0dc4046694d99caef633e92601ab52cbe17eb", size = 11925991, upload-time = "2025-10-21T14:51:56.535Z" }, - { url = "https://files.pythonhosted.org/packages/4d/3c/972edfddb4ae8a9fccd3c3766ed47453b6f805b6026b32f10209dd4b8ad4/cuda_bindings-12.9.4-cp313-cp313t-win_amd64.whl", hash = "sha256:b32d8b685f0e66f5658bcf4601ef034e89fc2843582886f0a58784a4302da06c", size = 11894363, upload-time = "2025-10-21T14:51:58.633Z" }, + { url = "https://files.pythonhosted.org/packages/90/21/ef85f3e15d394c9ca41fe116d78cd9e28533b9d7ead842f9241b332acf01/cuda_core-1.0.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9632db74eceb1cd72a7c95b61a5e4cfb9cc2291de0503e170334d936cab3316", size = 4788165, upload-time = "2026-05-12T20:11:17.116Z" }, + { url = "https://files.pythonhosted.org/packages/e0/41/c2c07b313c6cbb5d93010200c62b01ddb9f6c6f43a096a75c7b902c42ad6/cuda_core-1.0.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46690b0864417a5f2f9a7d10408e2570cbacae195c890a41286701eefb01ba79", size = 5061723, upload-time = "2026-05-12T20:11:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2d/b16b0af698a1bf4db337345daa7a44cd372fef107a3b692ffe1e0e6c5cc5/cuda_core-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1693fae113604cf9c114bfe3da15a15981f9d44ae78ecceb0b23e24c628ad19b", size = 4742003, upload-time = "2026-05-12T20:11:21.943Z" }, + { url = "https://files.pythonhosted.org/packages/41/4b/4ac1d0639241da756c634add606f93a7f3a39bef12f70e1fb4b40cc53c21/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3effd11283bc46fd06348c2fd18a0941ba7718a6f447343858c944c1a93a6dab", size = 4784340, upload-time = "2026-05-12T20:11:23.961Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/bb3e701f4af504e5e39e837135dc80022ec4c84858b2886ad577fe696a77/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1934517ff8a9dcd21b3f4a28e15e12643164b7d3ec187a4ee7560e22fd2dfc17", size = 5059041, upload-time = "2026-05-12T20:11:26.045Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e3/3ffaca2eabc71d0f9d29368fabc8ffb309353f05f418ea4c7eb5f223cf09/cuda_core-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:95c91d434a9baca066646cefa577227385104670a02fbe8e3defaadda84becf5", size = 4746198, upload-time = "2026-05-12T20:11:28.405Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a0/1daeae599cadd612689dbbf70d7da1c01883964fc2fbc7386f3c630a68cf/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1", size = 4789856, upload-time = "2026-05-12T20:11:30.862Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4d/603557ab3cb171cc2a61d3678a39cb4dae3fd21275078bfbd1c0b0b5230b/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2", size = 5106023, upload-time = "2026-05-12T20:11:33.509Z" }, + { url = "https://files.pythonhosted.org/packages/c2/1a/ae079963c9df7f4274227eb63cf8f6083a532a6443adb340d951fd21c626/cuda_core-1.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a5c1aa3b738a7599ea289498d038fe625d259fd7ab795394541eee58a8e29bc", size = 4663076, upload-time = "2026-05-12T20:11:35.784Z" }, + { url = "https://files.pythonhosted.org/packages/57/f9/a6676b1fa555fad5748a945f4b530b51b898b4771a1e5d9f3520d3f415ea/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c427e5025096d96fcd5092fdc85d5d5e4ac3dea007914e90472ed52f27220446", size = 4749800, upload-time = "2026-05-12T20:11:38.012Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9d/4534a9564a812ee95b43db7324f9b25cbffda001bb348bb5b3f90dad50b9/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b392178202c652368883dbe3773cee14f3e1ed6b8bf45d1a1bcdd37c73604e06", size = 5078597, upload-time = "2026-05-12T20:11:40.836Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7c/2f68b0bdeb7dd36204f752468254d6b4487c6d82e9e442cfbe815a656eac/cuda_core-1.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:b99e3ca9bf3bd2c7d3028e5dc541b00e432e21373816889cdf2722b675bd9be8", size = 4647545, upload-time = "2026-05-12T20:11:43.494Z" }, ] [[package]] @@ -622,13 +676,103 @@ wheels = [ name = "cuda-python" version = "12.9.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", +] dependencies = [ - { name = "cuda-bindings" }, + { name = "cuda-bindings", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/af/f3/6b032a554019cfb3447e671798c1bd3e79b5f1af20d10253f56cea269ef2/cuda_python-12.9.4-py3-none-any.whl", hash = "sha256:d2cacea882a69863f1e7d27ee71d75f0684f4c76910aff839067e4f89c902279", size = 7594, upload-time = "2025-10-21T14:55:12.846Z" }, ] +[[package]] +name = "cuda-python" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", +] +dependencies = [ + { name = "cuda-bindings", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "cuda-core", marker = "sys_platform != 'darwin'" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/31/7ff3f7768eded7535c621abc2fecb9d181a34ea4cae3afe682feb796f242/cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3", size = 8295, upload-time = "2026-05-29T23:28:47.012Z" }, +] + +[[package]] +name = "cuda-tile" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/1d/03499651b6957b31ab70c875d91f23220da5f8f84cbad1fb7cba2d7dd435/cuda_tile-1.5.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:27da6113d3469de0b2f75dd269904e730b6d2bc81336addbd299c593d6a125c4", size = 322958, upload-time = "2026-07-08T01:49:22.702Z" }, + { url = "https://files.pythonhosted.org/packages/d1/aa/8af16bde9b0c41cda286791749a246195716a15eea7f0dc79a99d37d8686/cuda_tile-1.5.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:cc390ac00d2ecd5f7c2e26198b410787fa39baf3737c6abceac9bda3ca9fccef", size = 324757, upload-time = "2026-07-08T01:49:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/50/ed/e98669f59bdea9d5f698cb22aab3bbc6ed10eadef640a79bf2d7e5a3c4cc/cuda_tile-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d808355e4cb6a850c7e3e82a5cd656c0062c33225fce69a8fd40290bc5b0bcaa", size = 305184, upload-time = "2026-07-08T01:49:28.779Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4d/e07fd65640c26c1f990ee621af11f073672e8d96501663026c7e1978f5b8/cuda_tile-1.5.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:81b8a93e757258260bd05dbbe6eec8bb50655ee1a88d5ebbc8412c526d3c0ed4", size = 322684, upload-time = "2026-07-08T01:49:21.318Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/981c2eb351f15a4b75b5913e35d2ef16cee32a45c950d0e16f85e626dd05/cuda_tile-1.5.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:9494170237d34bbbce83f2ada005d2dabb704f1f8c6a0af59088c180bd1bf028", size = 324278, upload-time = "2026-07-08T01:49:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/a863d460f6b70431869be5a18f82b1a1af91b5d7629e914c204e3545cac2/cuda_tile-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bda5f1721a6daa45124033f031685d1b30439ea9dbb07e0d46ec2355cfde0657", size = 304926, upload-time = "2026-07-08T01:49:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/7c/6d/cc2fb5a25689a501564a2eced4acf654f307e801a2c1506be97c0d100491/cuda_tile-1.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87652483baa9c81a9a24e4450f016e4ee78fd205d8422dad8996571bd1f2622e", size = 322641, upload-time = "2026-07-08T01:49:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/b4ba9d0fc71198d939ebf9a090228179995d8411ee9def8f638a0e3ccdc5/cuda_tile-1.5.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:cef6d30acc37557643ece0de3770fc4c33497c4af40209e424f72fbfcbe6ea5a", size = 324990, upload-time = "2026-07-08T01:49:17.739Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/7a60f317c503580ab7946dbb7fd080438fe953d0ddfdc81904beb9a1fab7/cuda_tile-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:16d97a60ed1d33388abbca85ea08cdae6325cc700476b3135f190d0fb50329f4", size = 304817, upload-time = "2026-07-08T01:49:38.853Z" }, + { url = "https://files.pythonhosted.org/packages/00/46/60aea981ee7cc0b159eb08c42b795e8e95ae8a7fb451e4565cad0f43cca0/cuda_tile-1.5.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:cfa4a5ef920d9c1fee702611b25bde449915f12bf929c1b8a3faee0c9b03f750", size = 322643, upload-time = "2026-07-08T01:49:26.107Z" }, + { url = "https://files.pythonhosted.org/packages/26/d5/ae03d2b70ed8d6c21ca809ddc98227ad07988e7fe67e7e41d888c0b13d32/cuda_tile-1.5.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:e7cb56186b0cc98166b72c7e5a3764c236151aa8d53e0b37a153766b79ea005a", size = 324995, upload-time = "2026-07-08T01:49:19.931Z" }, + { url = "https://files.pythonhosted.org/packages/da/a6/e3f6e9aefcc94d548e5d69f01a02ba5da7c5e200702eba3eb7a4f572a883/cuda_tile-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:c400918faa8492d84c4da3da81ce01ed30d81ed780f9ddbc1a9bdd588058b776", size = 304825, upload-time = "2026-07-08T01:49:37.749Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +curand = [ + { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusolver = [ + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] + [[package]] name = "depyf" version = "0.20.0" @@ -651,15 +795,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - [[package]] name = "distlib" version = "0.4.3" @@ -884,29 +1019,45 @@ wheels = [ ] [[package]] -name = "filelock" -version = "3.29.0" +name = "fastsafetensors" +version = "0.3.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +dependencies = [ + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/4c/f17bd54c933fd23648ce1e4272adf27d8d7e99b788c8859544bbd39f02e7/fastsafetensors-0.3.3.tar.gz", hash = "sha256:ba4fb59be8a6adbc91723848c3c6f57a9a9a5d2247d9768f84511380d01e554c", size = 77817, upload-time = "2026-07-07T07:21:47.121Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/75/06/dce623c5bc294dc72e4f8eb48d55a5db829b392b3e717a5156226be807b4/fastsafetensors-0.3.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db542d1dfa60dc3c610d9750e3d90107e2ef76feed34958ba8b81814969ea168", size = 1840879, upload-time = "2026-07-07T07:21:26.572Z" }, + { url = "https://files.pythonhosted.org/packages/14/28/cd0fe7ffe75a84ffe2d7da3e86bdb052f01f4b7442dde44e2837aa34e4ed/fastsafetensors-0.3.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50d8137ce2ca072d1717564c3810d7ccf649eb9f27179f26470258465682bf42", size = 1871721, upload-time = "2026-07-07T07:21:28.102Z" }, + { url = "https://files.pythonhosted.org/packages/65/e1/ed09e65cff92d93bf8b7ec52bd7912395df22b649105d906c5cf34be1c84/fastsafetensors-0.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:2f15b3d293468f2a6aae54b77781521a5d4f0db49bc6cd655442cca2207b3e78", size = 422851, upload-time = "2026-07-07T07:21:29.275Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/40bdc05f922251c54518232b2ef47fd925094ee699c93a45c8792628a10e/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df2f641647b79093c5e3f4bb0b9c7d771f22b61279714c3059d5c71a7cf708da", size = 1858970, upload-time = "2026-07-07T07:21:30.457Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f5/79f187385b4b9093742fec94e7889d22634ab6eba8403707797246fb2418/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92f8cf8e6617cffe24522023c726c5911c1ca5b7e3249da054abb0fac8d27040", size = 1891339, upload-time = "2026-07-07T07:21:32.022Z" }, + { url = "https://files.pythonhosted.org/packages/1f/36/e4e2f6bfaac6120e214624ead73ae12e304ad50e5dee7dbb249d9000f01f/fastsafetensors-0.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:be753c669b2e12b744f757303482d95079d48cd7fcce992680b74284b12e6ffa", size = 424283, upload-time = "2026-07-07T07:21:33.335Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/f95f7fc099ac1fc4c22aa46257d159eac88e29dd0765d21c4fc91caedb01/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c2f788a936ffd17938484360339645812e14cf1b2bdf4c18c035c713218e5a9", size = 1887326, upload-time = "2026-07-07T07:21:34.624Z" }, + { url = "https://files.pythonhosted.org/packages/92/8c/e3347b2a44a8ab9aced94fa450df4f309baa21f7f2981a8a7bd6a977f4d3/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3587bc66b8dec560ad903becf9540889013d4d47f0e10cf45f19bedc7b7bffa7", size = 1915478, upload-time = "2026-07-07T07:21:35.901Z" }, + { url = "https://files.pythonhosted.org/packages/80/65/388a55e6b2b3023fb732843335de14803f16a6d92f0cd47f1125d28b77ac/fastsafetensors-0.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:39c252a5528fa8653366f979d2ab8fa81159db4456b7c72cb5c288adb7699079", size = 424934, upload-time = "2026-07-07T07:21:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/16/a2/fd30fe6ed5825cb4642bdd102d3c5a32d2d82c91bdb45ae61a9571278cf8/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f969b2c818942748ff1bc619e6d575a7a7156a7d488d4c41779fba8c4e1891dc", size = 1887148, upload-time = "2026-07-07T07:21:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9d/d3db53dd5b5069b44f2b8f6fc5a943f4101178a6f74b48a7fc72dc68f142/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbf9fbc5f74d6a3f333777eb222aa63453795dacd7ae7f559316a5398d4942ef", size = 1915579, upload-time = "2026-07-07T07:21:40.047Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/4193c0ae305edb94a45f4b0789cf705106b87a1546ca5dafb211c8ab274a/fastsafetensors-0.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:b61a5f6e0e0b0da591f2790d3fbc6eef7f6e8597087977b946069e7bfbb2ec7a", size = 424967, upload-time = "2026-07-07T07:21:41.243Z" }, ] [[package]] -name = "flashinfer-cubin" -version = "0.6.6" +name = "filelock" +version = "3.29.0" source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/e8/826f9452bc5f76b94d7eb025f03dcaf1b51b9ed7790386c0285191e69be4/flashinfer_cubin-0.6.6-py3-none-any.whl", hash = "sha256:36508dfc792eb5ecfb15d2c140a7702812e1fa1ab0fb03929b2ed55e3e8191f3", size = 267661457, upload-time = "2026-03-11T01:36:36.538Z" }, + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] [[package]] name = "flashinfer-python" -version = "0.6.6" +version = "0.6.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, { name = "click" }, + { name = "cuda-tile" }, { name = "einops" }, { name = "ninja" }, { name = "numpy" }, @@ -919,9 +1070,9 @@ dependencies = [ { name = "torch" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/70/c5a235297351021f5d3d3233523a85f5a6468495587489ad2f257e8eafe2/flashinfer_python-0.6.6.tar.gz", hash = "sha256:0730ba7c7aad332961933bcebc5119762797161ede57d955f6fd199818ed1d92", size = 5344156, upload-time = "2026-03-11T01:36:21.434Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/11/ce2271271bee6990d34ed2d01288e9e92a0ea8ee45fb28de8e746c7da761/flashinfer_python-0.6.14.tar.gz", hash = "sha256:f4da8b5e005601784e85e0dcaa3389f908ee2d32c2560142d67124ab10e4a070", size = 9944949, upload-time = "2026-07-02T00:22:50.879Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/61/385d06755f3ab66333018285657adf0daf8a90a129448231fd09e315bd2e/flashinfer_python-0.6.6-py3-none-any.whl", hash = "sha256:078f158636969eec1a0d3dea19c3ca90b426b66df89bbf7b7b8276ce2ec08148", size = 7817047, upload-time = "2026-03-11T01:36:19.198Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8f/b101913cb2b3687654f56681cfe9836d447526be663c149966470ef70531/flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df", size = 14574383, upload-time = "2026-07-02T00:22:48.413Z" }, ] [[package]] @@ -1022,21 +1173,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] -[[package]] -name = "gguf" -version = "0.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/ae/17f1308ae45cd7b08ebb521747d5b23f4efc4d172038a4e228dd5106c3ff/gguf-0.19.0.tar.gz", hash = "sha256:dbadcd6cc7ccd44256f2229fe7c2dff5e8aa5cf0612ab987fd2b1a57e428923f", size = 111220, upload-time = "2026-05-06T13:04:03.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/bb/d71d6da82763528c2c2ed6b59a9d6142c6595545a4c448e2085d155e88c2/gguf-0.19.0-py3-none-any.whl", hash = "sha256:70bcd10edfe697fb2dad6e40af2234b9d8ece9a41a99761405121ebda1c3c1cd", size = 118475, upload-time = "2026-05-06T13:04:02.588Z" }, -] - [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -1226,6 +1362,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/a5/33b49ba7bea7c41bb37f74ec0f8beea0831e052330196633fe2c77516ea6/huggingface_hub-1.14.0-py3-none-any.whl", hash = "sha256:efe075535c62e130b30e836b138e13785f6f043d1f0539e0a39aa411a99e90b8", size = 661479, upload-time = "2026-05-06T14:14:32.029Z" }, ] +[[package]] +name = "humming-kernels" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "cuda-bindings", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "jinja2" }, + { name = "numpy" }, + { name = "nvidia-ml-py" }, + { name = "pyelftools" }, + { name = "safetensors" }, + { name = "tabulate" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "triton" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/4f/6977a31451c3f7aa1deaa76506d6cdeb74ef418ad8bcba2e98f7510b26ec/humming_kernels-0.1.10.tar.gz", hash = "sha256:da3e46fb9fc9eba2a9327c2e8135ead68e390c955acd7449f97ee7c71666c8b1", size = 220110, upload-time = "2026-07-02T10:22:57.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/ba/869bc24591d2b4fb0d8da821528072971052a934af077a11f77a0f2b3e79/humming_kernels-0.1.10-py3-none-any.whl", hash = "sha256:4ded0998ff085afeddde70baf93f97c2929969ec3d4a63a52cfec5072bc972b4", size = 184889, upload-time = "2026-07-02T10:22:56.031Z" }, +] + +[package.optional-dependencies] +cu13 = [ + { name = "nvidia-cuda-cccl" }, + { name = "nvidia-cuda-nvcc" }, + { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cuda-nvrtc", version = "13.3.33", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -1478,43 +1646,39 @@ wheels = [ [[package]] name = "llguidance" -version = "1.3.0" +version = "1.7.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/48/3f7a9d3ff1b36bba92b5107a3a21286821227afe9ea464736133994d61fb/llguidance-1.3.0.tar.gz", hash = "sha256:861249afd51dc325646834462ea827e57a5c2b2042e108e6aae7059fdad9104d", size = 1070460, upload-time = "2025-10-20T19:58:44.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/91/6bc8bb503dc259e46d253b5424385a54fe06c38a4c7a12befe69a3c2455a/llguidance-1.7.6.tar.gz", hash = "sha256:db7febbe412ed2015501904646750071d7e00e6df7f85c4b956ad4f206fd2df7", size = 1156574, upload-time = "2026-06-03T20:13:25.316Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/33/be5acb85cd8cdc4afde33d9c234eece9f318e087920255af3c05864cd3e7/llguidance-1.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7685222660a762e481ac633d49cc559c64980fe2ee59c8f932a5bb5cbc0c2c2", size = 3220647, upload-time = "2025-10-20T19:58:42.542Z" }, - { url = "https://files.pythonhosted.org/packages/82/e6/b48bda5b15efeaeb62bd0dba8fc6a01d4ae5457a85dbb5d18632385fe15c/llguidance-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:098030ff0687261a3f1bd54cf21fe951fc861d56d37a0671250dd36677eaf224", size = 3099830, upload-time = "2025-10-20T19:58:40.826Z" }, - { url = "https://files.pythonhosted.org/packages/aa/11/44389d3d1526d7a5c38ffd587a5ebc61d7bee443ac1dea95f2089ad58f5f/llguidance-1.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6caca5d78db7f76e1fbb0fff8607b861c32d47fa3d5dee2fc49de27ee269df", size = 2835242, upload-time = "2025-10-20T19:58:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/83/a8/1ff2bedb8f9acb46a2d2d603415d272bb622c142ea86f5b95445cc6e366c/llguidance-1.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc17e9dd602c3879bf91664a64bf72f54c74dbfbeb24ccfab6a5fe435b12f7aa", size = 3033133, upload-time = "2025-10-20T19:58:38.721Z" }, - { url = "https://files.pythonhosted.org/packages/5a/7e/809349638231f469b9056c0e1bfd924d5ef5558b3b3ec72d093b6fad33b1/llguidance-1.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:1d1cd1c8618d1a13605d3e057c978651e551c8c469b481ee4041f1d6c436002d", size = 2789946, upload-time = "2025-10-20T19:58:45.958Z" }, + { url = "https://files.pythonhosted.org/packages/fa/1d/5a9a13421b1f3f1c1acf82beb63ed72fa4d302e65099b72f4a4fe5a098ab/llguidance-1.7.6-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:eabf4572c8731734c0444c353b9ea06bc5c156986d2ff0a4ec0499159271381f", size = 3227892, upload-time = "2026-06-03T20:13:09.533Z" }, + { url = "https://files.pythonhosted.org/packages/46/fe/bb185f11bad82f2637e3cd8cbf6b200cbb6ed56ac395de47ea05a60d4649/llguidance-1.7.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:9c54c899db8cb4b4fba128a7d844730066576c70d806c95ada92b2bd2d6ab498", size = 3138127, upload-time = "2026-06-03T20:13:11.649Z" }, + { url = "https://files.pythonhosted.org/packages/51/b9/dc76d7716e04dc7b3427cae52eaa32bd20771382d4d1dd9f4538a9dd2086/llguidance-1.7.6-cp39-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:e70fa25ed550c2b50c2fd70baa9e2808b4ecb859d01e453bd5459aff62ba38c3", size = 2899993, upload-time = "2026-06-03T20:13:13.563Z" }, + { url = "https://files.pythonhosted.org/packages/1a/64/d74336f22242ef94356a456057d4ff1be7c1bc9c7dbc867171c6982a5512/llguidance-1.7.6-cp39-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:ceec951d29a74309984e3be0fe7f5f56c1362434cd937abd517b259a60908b1e", size = 3074809, upload-time = "2026-06-03T20:13:15.498Z" }, + { url = "https://files.pythonhosted.org/packages/49/37/99d700f0e2c83acf25a8d8946b2bee9f5eac47bc530bfbd53ba3126c667f/llguidance-1.7.6-cp39-abi3-win_amd64.whl", hash = "sha256:ace7e81cd31950a87186356ab24bd7f75fbc10a05ca9d9f7f8748f931963f763", size = 2879207, upload-time = "2026-06-03T20:13:23.341Z" }, ] [[package]] name = "llvmlite" -version = "0.44.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/89/6a/95a3d3610d5c75293d5dbbb2a76480d5d4eeba641557b69fe90af6c5b84e/llvmlite-0.44.0.tar.gz", hash = "sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4", size = 171880, upload-time = "2025-01-20T11:14:41.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/75/d4863ddfd8ab5f6e70f4504cf8cc37f4e986ec6910f4ef8502bb7d3c1c71/llvmlite-0.44.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9fbadbfba8422123bab5535b293da1cf72f9f478a65645ecd73e781f962ca614", size = 28132306, upload-time = "2025-01-20T11:12:18.634Z" }, - { url = "https://files.pythonhosted.org/packages/37/d9/6e8943e1515d2f1003e8278819ec03e4e653e2eeb71e4d00de6cfe59424e/llvmlite-0.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cccf8eb28f24840f2689fb1a45f9c0f7e582dd24e088dcf96e424834af11f791", size = 26201096, upload-time = "2025-01-20T11:12:24.544Z" }, - { url = "https://files.pythonhosted.org/packages/aa/46/8ffbc114def88cc698906bf5acab54ca9fdf9214fe04aed0e71731fb3688/llvmlite-0.44.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7202b678cdf904823c764ee0fe2dfe38a76981f4c1e51715b4cb5abb6cf1d9e8", size = 42361859, upload-time = "2025-01-20T11:12:31.839Z" }, - { url = "https://files.pythonhosted.org/packages/30/1c/9366b29ab050a726af13ebaae8d0dff00c3c58562261c79c635ad4f5eb71/llvmlite-0.44.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40526fb5e313d7b96bda4cbb2c85cd5374e04d80732dd36a282d72a560bb6408", size = 41184199, upload-time = "2025-01-20T11:12:40.049Z" }, - { url = "https://files.pythonhosted.org/packages/69/07/35e7c594b021ecb1938540f5bce543ddd8713cff97f71d81f021221edc1b/llvmlite-0.44.0-cp310-cp310-win_amd64.whl", hash = "sha256:41e3839150db4330e1b2716c0be3b5c4672525b4c9005e17c7597f835f351ce2", size = 30332381, upload-time = "2025-01-20T11:12:47.054Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e2/86b245397052386595ad726f9742e5223d7aea999b18c518a50e96c3aca4/llvmlite-0.44.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3", size = 28132305, upload-time = "2025-01-20T11:12:53.936Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ec/506902dc6870249fbe2466d9cf66d531265d0f3a1157213c8f986250c033/llvmlite-0.44.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ace564d9fa44bb91eb6e6d8e7754977783c68e90a471ea7ce913bff30bd62427", size = 26201090, upload-time = "2025-01-20T11:12:59.847Z" }, - { url = "https://files.pythonhosted.org/packages/99/fe/d030f1849ebb1f394bb3f7adad5e729b634fb100515594aca25c354ffc62/llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5d22c3bfc842668168a786af4205ec8e3ad29fb1bc03fd11fd48460d0df64c1", size = 42361858, upload-time = "2025-01-20T11:13:07.623Z" }, - { url = "https://files.pythonhosted.org/packages/d7/7a/ce6174664b9077fc673d172e4c888cb0b128e707e306bc33fff8c2035f0d/llvmlite-0.44.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610", size = 41184200, upload-time = "2025-01-20T11:13:20.058Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c6/258801143975a6d09a373f2641237992496e15567b907a4d401839d671b8/llvmlite-0.44.0-cp311-cp311-win_amd64.whl", hash = "sha256:d8489634d43c20cd0ad71330dde1d5bc7b9966937a263ff1ec1cebb90dc50955", size = 30331193, upload-time = "2025-01-20T11:13:26.976Z" }, - { url = "https://files.pythonhosted.org/packages/15/86/e3c3195b92e6e492458f16d233e58a1a812aa2bfbef9bdd0fbafcec85c60/llvmlite-0.44.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:1d671a56acf725bf1b531d5ef76b86660a5ab8ef19bb6a46064a705c6ca80aad", size = 28132297, upload-time = "2025-01-20T11:13:32.57Z" }, - { url = "https://files.pythonhosted.org/packages/d6/53/373b6b8be67b9221d12b24125fd0ec56b1078b660eeae266ec388a6ac9a0/llvmlite-0.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f79a728e0435493611c9f405168682bb75ffd1fbe6fc360733b850c80a026db", size = 26201105, upload-time = "2025-01-20T11:13:38.744Z" }, - { url = "https://files.pythonhosted.org/packages/cb/da/8341fd3056419441286c8e26bf436923021005ece0bff5f41906476ae514/llvmlite-0.44.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0143a5ef336da14deaa8ec26c5449ad5b6a2b564df82fcef4be040b9cacfea9", size = 42361901, upload-time = "2025-01-20T11:13:46.711Z" }, - { url = "https://files.pythonhosted.org/packages/53/ad/d79349dc07b8a395a99153d7ce8b01d6fcdc9f8231355a5df55ded649b61/llvmlite-0.44.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d752f89e31b66db6f8da06df8b39f9b91e78c5feea1bf9e8c1fba1d1c24c065d", size = 41184247, upload-time = "2025-01-20T11:13:56.159Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3b/a9a17366af80127bd09decbe2a54d8974b6d8b274b39bf47fbaedeec6307/llvmlite-0.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:eae7e2d4ca8f88f89d315b48c6b741dcb925d6a1042da694aa16ab3dd4cbd3a1", size = 30332380, upload-time = "2025-01-20T11:14:02.442Z" }, - { url = "https://files.pythonhosted.org/packages/89/24/4c0ca705a717514c2092b18476e7a12c74d34d875e05e4d742618ebbf449/llvmlite-0.44.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:319bddd44e5f71ae2689859b7203080716448a3cd1128fb144fe5c055219d516", size = 28132306, upload-time = "2025-01-20T11:14:09.035Z" }, - { url = "https://files.pythonhosted.org/packages/01/cf/1dd5a60ba6aee7122ab9243fd614abcf22f36b0437cbbe1ccf1e3391461c/llvmlite-0.44.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c58867118bad04a0bb22a2e0068c693719658105e40009ffe95c7000fcde88e", size = 26201090, upload-time = "2025-01-20T11:14:15.401Z" }, - { url = "https://files.pythonhosted.org/packages/d2/1b/656f5a357de7135a3777bd735cc7c9b8f23b4d37465505bd0eaf4be9befe/llvmlite-0.44.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46224058b13c96af1365290bdfebe9a6264ae62fb79b2b55693deed11657a8bf", size = 42361904, upload-time = "2025-01-20T11:14:22.949Z" }, - { url = "https://files.pythonhosted.org/packages/d8/e1/12c5f20cb9168fb3464a34310411d5ad86e4163c8ff2d14a2b57e5cc6bac/llvmlite-0.44.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0097052c32bf721a4efc03bd109d335dfa57d9bffb3d4c24cc680711b8b4fc", size = 41184245, upload-time = "2025-01-20T11:14:31.731Z" }, - { url = "https://files.pythonhosted.org/packages/d0/81/e66fc86539293282fd9cb7c9417438e897f369e79ffb62e1ae5e5154d4dd/llvmlite-0.44.0-cp313-cp313-win_amd64.whl", hash = "sha256:2fb7c4f2fb86cbae6dca3db9ab203eeea0e22d73b99bc2341cdf9de93612e930", size = 30331193, upload-time = "2025-01-20T11:14:38.578Z" }, +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/f5/a1bde3aa8c43524b0acaf3f72fb3d80a32dd29dbb42d7dc434f84584cdcc/llvmlite-0.47.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41270b0b1310717f717cf6f2a9c68d3c43bd7905c33f003825aebc361d0d1b17", size = 37232772, upload-time = "2026-03-31T18:28:12.198Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fb/76d88fc05ee1f9c1a6efe39eb493c4a727e5d1690412469017cd23bcb776/llvmlite-0.47.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f9d118bc1dd7623e0e65ca9ac485ec6dd543c3b77bc9928ddc45ebd34e1e30a7", size = 56275179, upload-time = "2026-03-31T18:28:15.725Z" }, + { url = "https://files.pythonhosted.org/packages/4d/08/29da7f36217abd56a0c389ef9a18bea47960826e691ced1a36c92c6ce93c/llvmlite-0.47.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ea5cfb04a6ab5b18e46be72b41b015975ba5980c4ddb41f1975b83e19031063", size = 55128632, upload-time = "2026-03-31T18:28:19.946Z" }, + { url = "https://files.pythonhosted.org/packages/df/f8/5e12e9ed447d65f04acf6fcf2d79cded2355640b5131a46cee4c99a5949d/llvmlite-0.47.0-cp310-cp310-win_amd64.whl", hash = "sha256:166b896a2262a2039d5fc52df5ee1659bd1ccd081183df7a2fba1b74702dd5ea", size = 38138402, upload-time = "2026-03-31T18:28:23.327Z" }, + { url = "https://files.pythonhosted.org/packages/34/0b/b9d1911cfefa61399821dfb37f486d83e0f42630a8d12f7194270c417002/llvmlite-0.47.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:74090f0dcfd6f24ebbef3f21f11e38111c4d7e6919b54c4416e1e357c3446b07", size = 37232770, upload-time = "2026-03-31T18:28:26.765Z" }, + { url = "https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca14f02e29134e837982497959a8e2193d6035235de1cb41a9cb2bd6da4eedbb", size = 56275177, upload-time = "2026-03-31T18:28:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/7e/51/48a53fedf01cb1f3f43ef200be17ebf83c8d9a04018d3783c1a226c342c2/llvmlite-0.47.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12a69d4bb05f402f30477e21eeabe81911e7c251cecb192bed82cd83c9db10d8", size = 55128631, upload-time = "2026-03-31T18:28:36.046Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/59227d06bdc96e23322713c381af4e77420949d8cd8a042c79e0043096cc/llvmlite-0.47.0-cp311-cp311-win_amd64.whl", hash = "sha256:c37d6eb7aaabfa83ab9c2ff5b5cdb95a5e6830403937b2c588b7490724e05327", size = 38138400, upload-time = "2026-03-31T18:28:40.076Z" }, + { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, + { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, + { url = "https://files.pythonhosted.org/packages/77/6f/4615353e016799f80fa52ccb270a843c413b22361fadda2589b2922fb9b0/llvmlite-0.47.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a3c6a735d4e1041808434f9d440faa3d78d9b4af2ee64d05a66f351883b6ceec", size = 37232771, upload-time = "2026-03-31T18:29:01.324Z" }, + { url = "https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2699a74321189e812d476a43d6d7f652f51811e7b5aad9d9bba842a1c7927acb", size = 56275178, upload-time = "2026-03-31T18:29:05.748Z" }, + { url = "https://files.pythonhosted.org/packages/d6/da/b32cafcb926fb0ce2aa25553bf32cb8764af31438f40e2481df08884c947/llvmlite-0.47.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c6951e2b29930227963e53ee152441f0e14be92e9d4231852102d986c761e40", size = 55128632, upload-time = "2026-03-31T18:29:11.235Z" }, + { url = "https://files.pythonhosted.org/packages/46/9f/4898b44e4042c60fafcb1162dfb7014f6f15b1ec19bf29cfea6bf26df90d/llvmlite-0.47.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2e9adf8698d813a9a5efb2d4370caf344dbc1e145019851fee6a6f319ba760e", size = 38138695, upload-time = "2026-03-31T18:29:15.43Z" }, ] [[package]] @@ -1656,7 +1820,7 @@ wheels = [ [[package]] name = "mistral-common" -version = "1.11.2" +version = "1.11.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, @@ -1668,9 +1832,9 @@ dependencies = [ { name = "tiktoken" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/eb/12167a1bea9714582e5b4f539f9c019323363e314a499c72855ff0e5ad43/mistral_common-1.11.2.tar.gz", hash = "sha256:79f68fc2d1190f28637f40e053f919c8c2697e00b2aa679ddee562a95183f4ad", size = 6357845, upload-time = "2026-05-04T19:47:40.413Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/d0/61b2c24be62a8e2f0e46a1c16de23de386c8644408da249bc66768a6681b/mistral_common-1.11.7.tar.gz", hash = "sha256:d3b79583595cf6d96a2ab33e42cb8449768383147b8c56cac5a4f193be19d20d", size = 6387178, upload-time = "2026-07-23T09:21:17.206Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/f0/6a5d604b972e442b9d36c117d01788feddad099e4965699e3516ee6fefc3/mistral_common-1.11.2-py3-none-any.whl", hash = "sha256:ebb42062cd705a0aa2bc69b4cde2b83d446ae58150b7e29322c90cb08fcfca6c", size = 6531968, upload-time = "2026-05-04T19:47:37.718Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a4/bc2850eb33cc2d633a21f51530756350dca325ee69b07c9202552f2bbadb/mistral_common-1.11.7-py3-none-any.whl", hash = "sha256:a9511b88eacacbe7dacddd9d3498c1739f56847b7fdddbd5a22e7844fd9def95", size = 6553583, upload-time = "2026-07-23T09:21:19.818Z" }, ] [package.optional-dependencies] @@ -1678,6 +1842,41 @@ image = [ { name = "opencv-python-headless" }, ] +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/3a/c5b855752a70267ff729c349e650263adb3c206c29d28cc8ea7ace30a1d5/ml_dtypes-0.5.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c", size = 679735, upload-time = "2025-11-17T22:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/41/79/7433f30ee04bd4faa303844048f55e1eb939131c8e5195a00a96a0939b64/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a", size = 5051883, upload-time = "2025-11-17T22:31:33.658Z" }, + { url = "https://files.pythonhosted.org/packages/10/b1/8938e8830b0ee2e167fc75a094dea766a1152bde46752cd9bfc57ee78a82/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270", size = 5030369, upload-time = "2025-11-17T22:31:35.595Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a3/51886727bd16e2f47587997b802dd56398692ce8c6c03c2e5bb32ecafe26/ml_dtypes-0.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2", size = 210738, upload-time = "2025-11-17T22:31:37.43Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/712092cfe7e5eb667b8ad9ca7c54442f21ed7ca8979745f1000e24cf8737/ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90", size = 679734, upload-time = "2025-11-17T22:31:39.223Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/912146dfd4b5c0eea956836c01dcd2fce6c9c844b2691f5152aca196ce4f/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040", size = 5056165, upload-time = "2025-11-17T22:31:41.071Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/19189ea605017473660e43762dc853d2797984b3c7bf30ce656099add30c/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483", size = 5034975, upload-time = "2025-11-17T22:31:42.758Z" }, + { url = "https://files.pythonhosted.org/packages/b4/24/70bd59276883fdd91600ca20040b41efd4902a923283c4d6edcb1de128d2/ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb", size = 210742, upload-time = "2025-11-17T22:31:44.068Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c9/64230ef14e40aa3f1cb254ef623bf812735e6bec7772848d19131111ac0d/ml_dtypes-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de", size = 160709, upload-time = "2025-11-17T22:31:46.557Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, +] + [[package]] name = "model-hosting-container-standards" version = "0.1.15" @@ -1852,7 +2051,8 @@ name = "networkx" version = "3.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } wheels = [ @@ -1864,9 +2064,12 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ @@ -1910,34 +2113,30 @@ wheels = [ [[package]] name = "numba" -version = "0.61.2" +version = "0.65.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "llvmlite" }, { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/a0/e21f57604304aa03ebb8e098429222722ad99176a4f979d34af1d1ee80da/numba-0.61.2.tar.gz", hash = "sha256:8750ee147940a6637b80ecf7f95062185ad8726c8c28a2295b8ec1160a196f7d", size = 2820615, upload-time = "2025-04-09T02:58:07.659Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/ca/f470be59552ccbf9531d2d383b67ae0b9b524d435fb4a0d229fef135116e/numba-0.61.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cf9f9fc00d6eca0c23fc840817ce9f439b9f03c8f03d6246c0e7f0cb15b7162a", size = 2775663, upload-time = "2025-04-09T02:57:34.143Z" }, - { url = "https://files.pythonhosted.org/packages/f5/13/3bdf52609c80d460a3b4acfb9fdb3817e392875c0d6270cf3fd9546f138b/numba-0.61.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd", size = 2778344, upload-time = "2025-04-09T02:57:36.609Z" }, - { url = "https://files.pythonhosted.org/packages/e2/7d/bfb2805bcfbd479f04f835241ecf28519f6e3609912e3a985aed45e21370/numba-0.61.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ae8c7a522c26215d5f62ebec436e3d341f7f590079245a2f1008dfd498cc1642", size = 3824054, upload-time = "2025-04-09T02:57:38.162Z" }, - { url = "https://files.pythonhosted.org/packages/e3/27/797b2004745c92955470c73c82f0e300cf033c791f45bdecb4b33b12bdea/numba-0.61.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd1e74609855aa43661edffca37346e4e8462f6903889917e9f41db40907daa2", size = 3518531, upload-time = "2025-04-09T02:57:39.709Z" }, - { url = "https://files.pythonhosted.org/packages/b1/c6/c2fb11e50482cb310afae87a997707f6c7d8a48967b9696271347441f650/numba-0.61.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae45830b129c6137294093b269ef0a22998ccc27bf7cf096ab8dcf7bca8946f9", size = 2831612, upload-time = "2025-04-09T02:57:41.559Z" }, - { url = "https://files.pythonhosted.org/packages/3f/97/c99d1056aed767503c228f7099dc11c402906b42a4757fec2819329abb98/numba-0.61.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:efd3db391df53aaa5cfbee189b6c910a5b471488749fd6606c3f33fc984c2ae2", size = 2775825, upload-time = "2025-04-09T02:57:43.442Z" }, - { url = "https://files.pythonhosted.org/packages/95/9e/63c549f37136e892f006260c3e2613d09d5120672378191f2dc387ba65a2/numba-0.61.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:49c980e4171948ffebf6b9a2520ea81feed113c1f4890747ba7f59e74be84b1b", size = 2778695, upload-time = "2025-04-09T02:57:44.968Z" }, - { url = "https://files.pythonhosted.org/packages/97/c8/8740616c8436c86c1b9a62e72cb891177d2c34c2d24ddcde4c390371bf4c/numba-0.61.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3945615cd73c2c7eba2a85ccc9c1730c21cd3958bfcf5a44302abae0fb07bb60", size = 3829227, upload-time = "2025-04-09T02:57:46.63Z" }, - { url = "https://files.pythonhosted.org/packages/fc/06/66e99ae06507c31d15ff3ecd1f108f2f59e18b6e08662cd5f8a5853fbd18/numba-0.61.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbfdf4eca202cebade0b7d43896978e146f39398909a42941c9303f82f403a18", size = 3523422, upload-time = "2025-04-09T02:57:48.222Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a4/2b309a6a9f6d4d8cfba583401c7c2f9ff887adb5d54d8e2e130274c0973f/numba-0.61.2-cp311-cp311-win_amd64.whl", hash = "sha256:76bcec9f46259cedf888041b9886e257ae101c6268261b19fda8cfbc52bec9d1", size = 2831505, upload-time = "2025-04-09T02:57:50.108Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a0/c6b7b9c615cfa3b98c4c63f4316e3f6b3bbe2387740277006551784218cd/numba-0.61.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2", size = 2776626, upload-time = "2025-04-09T02:57:51.857Z" }, - { url = "https://files.pythonhosted.org/packages/92/4a/fe4e3c2ecad72d88f5f8cd04e7f7cff49e718398a2fac02d2947480a00ca/numba-0.61.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ddce10009bc097b080fc96876d14c051cc0c7679e99de3e0af59014dab7dfe8", size = 2779287, upload-time = "2025-04-09T02:57:53.658Z" }, - { url = "https://files.pythonhosted.org/packages/9a/2d/e518df036feab381c23a624dac47f8445ac55686ec7f11083655eb707da3/numba-0.61.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b1bb509d01f23d70325d3a5a0e237cbc9544dd50e50588bc581ba860c213546", size = 3885928, upload-time = "2025-04-09T02:57:55.206Z" }, - { url = "https://files.pythonhosted.org/packages/10/0f/23cced68ead67b75d77cfcca3df4991d1855c897ee0ff3fe25a56ed82108/numba-0.61.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:48a53a3de8f8793526cbe330f2a39fe9a6638efcbf11bd63f3d2f9757ae345cd", size = 3577115, upload-time = "2025-04-09T02:57:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/68/1d/ddb3e704c5a8fb90142bf9dc195c27db02a08a99f037395503bfbc1d14b3/numba-0.61.2-cp312-cp312-win_amd64.whl", hash = "sha256:97cf4f12c728cf77c9c1d7c23707e4d8fb4632b46275f8f3397de33e5877af18", size = 2831929, upload-time = "2025-04-09T02:57:58.45Z" }, - { url = "https://files.pythonhosted.org/packages/0b/f3/0fe4c1b1f2569e8a18ad90c159298d862f96c3964392a20d74fc628aee44/numba-0.61.2-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:3a10a8fc9afac40b1eac55717cece1b8b1ac0b946f5065c89e00bde646b5b154", size = 2771785, upload-time = "2025-04-09T02:57:59.96Z" }, - { url = "https://files.pythonhosted.org/packages/e9/71/91b277d712e46bd5059f8a5866862ed1116091a7cb03bd2704ba8ebe015f/numba-0.61.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d3bcada3c9afba3bed413fba45845f2fb9cd0d2b27dd58a1be90257e293d140", size = 2773289, upload-time = "2025-04-09T02:58:01.435Z" }, - { url = "https://files.pythonhosted.org/packages/0d/e0/5ea04e7ad2c39288c0f0f9e8d47638ad70f28e275d092733b5817cf243c9/numba-0.61.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bdbca73ad81fa196bd53dc12e3aaf1564ae036e0c125f237c7644fe64a4928ab", size = 3893918, upload-time = "2025-04-09T02:58:02.933Z" }, - { url = "https://files.pythonhosted.org/packages/17/58/064f4dcb7d7e9412f16ecf80ed753f92297e39f399c905389688cf950b81/numba-0.61.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f154aaea625fb32cfbe3b80c5456d514d416fcdf79733dd69c0df3a11348e9e", size = 3584056, upload-time = "2025-04-09T02:58:04.538Z" }, - { url = "https://files.pythonhosted.org/packages/af/a4/6d3a0f2d3989e62a18749e1e9913d5fa4910bbb3e3311a035baea6caf26d/numba-0.61.2-cp313-cp313-win_amd64.whl", hash = "sha256:59321215e2e0ac5fa928a8020ab00b8e57cda8a97384963ac0dfa4d4e6aa54e7", size = 2831846, upload-time = "2025-04-09T02:58:06.125Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/9b/e8453d93d5cb3f53cc956f135024be09d52f4f99643acaf8fdca090a8f3c/numba-0.65.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:dff9fd5fbc9a35c517359c5823ea705d9b65f01fb46e42e35a2eabe5a52c2e96", size = 2680537, upload-time = "2026-04-01T03:51:17.325Z" }, + { url = "https://files.pythonhosted.org/packages/07/95/d6a2f0625e1092624228301eea11cdaff21ddcaf917ef3d631846a38b2f4/numba-0.65.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c894c94afa5ffd627c7e3b693df10cb0d905bd5eb06de3dfc31775140cf4f89", size = 3739444, upload-time = "2026-04-01T03:51:19.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/fe518c97af035e4ec670c2edc3f0ff7a518cbed2f0b5053124d7c979bd8a/numba-0.65.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7325b1aab88f0339057288ee32f39dc660e14f93872a6fda14fa6eb9f95b047", size = 3446390, upload-time = "2026-04-01T03:51:21.55Z" }, + { url = "https://files.pythonhosted.org/packages/d0/06/5010939854249c290c6217e3fb7404914f4ed953f9923e340c3e166bcaf0/numba-0.65.0-cp310-cp310-win_amd64.whl", hash = "sha256:71e72e9ca2f619df4768f9c3962bfec60191a5a26fe2b6a8c6a07532b6146169", size = 2747200, upload-time = "2026-04-01T03:51:23.674Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ce/d67c499703eb5479ce02420e8ccd65c5753d87d2e16d563f152d71405346/numba-0.65.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:28e547d0b18024f19cbaf9de02fc5c145790213d9be8a2c95b43f93ec162b9e4", size = 2680228, upload-time = "2026-04-01T03:51:25.401Z" }, + { url = "https://files.pythonhosted.org/packages/c1/a7/11e2b24251d57cf41fc9ad83f378d890d61a890e3f8eb6338b39833f67a4/numba-0.65.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:032b0b8e879512cd424d79eed6d772a1399c6387ded184c2cf3cc22c08d750a6", size = 3744674, upload-time = "2026-04-01T03:51:27.311Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0b/7c63eb742859a6243f42288441f65ac9dac96ea59f409e43b713aafbe867/numba-0.65.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af143d823624033a128b5950c0aaf9ffc2386dfe954eb757119cf0432335534c", size = 3450620, upload-time = "2026-04-01T03:51:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/53/ff/1371cbbe955be340a46093a10b61462437e0fadc7a63290473a0e584cb03/numba-0.65.0-cp311-cp311-win_amd64.whl", hash = "sha256:15d159578e59a39df246b83480f78d7794b0fca40153b5684d3849a99c48a0fb", size = 2747081, upload-time = "2026-04-01T03:51:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, + { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, + { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, + { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f8/eee0f1ff456218db036bfc9023995ec1f85a9dc8f2422f1594f6a87829e0/numba-0.65.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c6334094563a456a695c812e6846288376ca02327cf246cdcc83e1bb27862367", size = 2680679, upload-time = "2026-04-01T03:51:39.491Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8f/3d116e4b8e92f6abace431afa4b2b944f4d65bdee83af886f5c4b263df95/numba-0.65.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b8a9008411615c69d083d1dcf477f75a5aa727b30beb16e139799e2be945cdfd", size = 3809537, upload-time = "2026-04-01T03:51:41.42Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/6a3ca4128e253cb67affe06deb47688f51ce968f5111e2a06d010e6f1fa6/numba-0.65.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af96c0cba53664efcb361528b8c75e011a6556c859c7e08424c2715201c6cf7a", size = 3508615, upload-time = "2026-04-01T03:51:43.444Z" }, + { url = "https://files.pythonhosted.org/packages/96/0e/267f9a36fb282c104a971d7eecb685b411c47dce2a740fe69cf5fc2945d9/numba-0.65.0-cp313-cp313-win_amd64.whl", hash = "sha256:6254e73b9c929dc736a1fbd3d6f5680789709a5067cae1fa7198707385129c04", size = 2749938, upload-time = "2026-04-01T03:51:45.218Z" }, ] [[package]] @@ -2003,155 +2202,322 @@ wheels = [ ] [[package]] -name = "nvidia-cublas-cu12" -version = "12.8.4.1" +name = "nvidia-cublas" +version = "13.1.0.3" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, ] [[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.8.90" +name = "nvidia-cuda-cccl" +version = "13.3.3.4.1" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, + { url = "https://files.pythonhosted.org/packages/96/bd/572971ffc14bd36676c821fc15d991b08fe6179cb09368250147475f954d/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:067d19b4b3c9d0f2ebec9f29a311b2863db96bf98e058bbc331597d51ce818cf", size = 3454030, upload-time = "2026-06-29T16:41:49.092Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ab/049726d90147865a3ea53bae6cb7c35b98bf1fdf96cdb967101329625f83/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc0adc188d570b09f4d606c7dc05a42aa3d8aa082e0d60f7bbfc5b6435f627c6", size = 3454034, upload-time = "2026-06-29T16:42:07.435Z" }, + { url = "https://files.pythonhosted.org/packages/24/d3/b1afcd9c40ceca72022579215fcaf5318cd747fd896cb928d4a1de924ff8/nvidia_cuda_cccl-13.3.3.4.1-py3-none-win_amd64.whl", hash = "sha256:d7c92cc03047031fa7af30866636d35ce4af409c28fc7dd8f69cb17053741399", size = 3454014, upload-time = "2026-06-29T17:09:09.012Z" }, ] [[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.8.93" +name = "nvidia-cuda-crt" +version = "13.3.73" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/fa/41/2089e411507d66458d67208bdd1bc562d492bb6458c3d2aea4603072a219/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:60aacc0b5e1e8b40c62abe4d1ab16440add91b99bd2f17f62dd091586b73d166", size = 157353, upload-time = "2026-06-29T16:42:38.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ce/16d76f4b5b3f7460f5ebd17516685495c149c66651ffdd381f90e4d4e65c/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df14a17ae1c5c3171265411212246654d780f89344ea85344466c6b955247543", size = 157352, upload-time = "2026-06-29T16:43:09.209Z" }, + { url = "https://files.pythonhosted.org/packages/49/b3/6791ffba6f4b8e0d3ed875285aad8078ee407afa464ecd934ae298c205b1/nvidia_cuda_crt-13.3.73-py3-none-win_amd64.whl", hash = "sha256:af04e75148db1f0eea30958f33a9ec5a5a2dc2afa99ca4323f9a93b840602ca5", size = 158286, upload-time = "2026-06-29T17:09:28.621Z" }, ] [[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.8.90" +name = "nvidia-cuda-cupti" +version = "13.0.85" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, ] [[package]] -name = "nvidia-cudnn-cu12" -version = "9.10.2.21" +name = "nvidia-cuda-nvcc" +version = "13.3.73" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cuda-crt" }, + { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "nvidia-nvvm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/14/9f5cdc994d5431e2f08f62ffe34509e7feabd1f2e18517e2d7720c6ff0fd/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70f250825355d2c3aa6c7a972a0ec00f020bad66d2679e527eb4336301c904aa", size = 39515578, upload-time = "2026-06-29T16:47:40.318Z" }, + { url = "https://files.pythonhosted.org/packages/83/19/e46ef3597ba47a9f8a91ab24533db42a600b659fc418dbe4af0b630bcb41/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f483af83166c4fa356a21606076d553b0b4ceaebbd9912e537545080db695bdd", size = 44942138, upload-time = "2026-06-29T16:48:13.615Z" }, + { url = "https://files.pythonhosted.org/packages/79/89/97eb797bb8bdee1d4e74069d072c24b79ae90c012fa3b539f2a7ccecf6cf/nvidia_cuda_nvcc-13.3.73-py3-none-win_amd64.whl", hash = "sha256:3d9da631bcac3dee49d1357b84cd05abe56aa3ccf76b05a7df8a80ef78addcb5", size = 32536529, upload-time = "2026-06-29T17:11:25.455Z" }, +] + +[[package]] +name = "nvidia-cuda-nvdisasm" +version = "13.3.73" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/be/e9de501cb71b10f7654381a485fa4ebf470ea25c3dce018cccaecf8a8f9a/nvidia_cuda_nvdisasm-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dd4751884f9016b9b6dbf007abdeb5681d0a2edc731dd3d2fda9d6d878e88f73", size = 4744517, upload-time = "2026-06-29T16:48:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/86/3e/88460ebd737e559e8e9843db7a63f8ced9ec7be1882344438819dd13aebc/nvidia_cuda_nvdisasm-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa17084b07c0dca68a42892f771b4b1b40fbe9b91660209623e61cea611cae8c", size = 4782824, upload-time = "2026-06-29T16:49:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/32/63/00d687730b124f94345f83023363251310ccc08eeac269ec2eeff6b097f3/nvidia_cuda_nvdisasm-13.3.73-py3-none-win_amd64.whl", hash = "sha256:da2fab133c3d095d83f13587eb87149beabd199b34a3cc270d0aa99449a628c3", size = 5015368, upload-time = "2026-06-29T17:22:50.207Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/4a/af/345fedb9f4c76c84ab4fa445b36bd4048a4d9db60e6bc76b4f913ff4b852/nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872", size = 76807835, upload-time = "2025-09-04T08:39:15.274Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.3.29" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, ] [[package]] name = "nvidia-cudnn-frontend" -version = "1.18.0" +version = "1.26.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/86/be/f5a1e633c524c13c0182213ab27dab42dca29a3c785be5ff74d2d185aed1/nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:baa6fbc8e7c55f1c78c0374ed9a890e1cf81acaca0c92d6135d18a8e3c985244", size = 2023500, upload-time = "2026-01-27T23:31:34.747Z" }, - { url = "https://files.pythonhosted.org/packages/82/a7/765a17c6a9496196c34f269d17dfb902b6c618c0261c0962511e95302e81/nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4bcca42259e358002c8867e3624a558f66cd5dff2cc6c3aafd860ef2f41730", size = 2154278, upload-time = "2026-01-27T23:06:55.784Z" }, - { url = "https://files.pythonhosted.org/packages/19/a1/7caae2243540bc60e47eae95f0fd913c9baa05cf94df0471914f70d45158/nvidia_cudnn_frontend-1.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:06252021ef1e5a7256f1e70429a426b01792636c05cc547fe8e64c6885a9652e", size = 1590158, upload-time = "2026-01-27T23:08:26.703Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9a/83d3d080118de4a7810fa019349edec634b8b37b9cafaacd05719de62dd6/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6d4d0b88d617b233a503c84980b54d840b60b2734497d1a7a071ec5293daec2", size = 2023709, upload-time = "2026-01-27T23:32:10.912Z" }, - { url = "https://files.pythonhosted.org/packages/13/c7/c3624b3ed77b102618f26295e816b27f1c3ebb1143730237a9f51d403c3f/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382ea063b92cbfd5b442cb75ff8422932d78276aecf139e46713ed1ad3d07af4", size = 2155568, upload-time = "2026-01-27T23:07:13.277Z" }, - { url = "https://files.pythonhosted.org/packages/52/dd/8613dfd029d076b86a8a87efe3f4bb4ab73cec15fa8fc27e665098f4d167/nvidia_cudnn_frontend-1.18.0-cp311-cp311-win_amd64.whl", hash = "sha256:baa509effc4d299d3f04e549d4188f88bca8a8b527f483cbd2f66bc18f13a8b1", size = 1591244, upload-time = "2026-01-27T23:08:44.691Z" }, - { url = "https://files.pythonhosted.org/packages/e3/b4/604e230378680ee117849a4e1045baca092f93161a829291a84d5acce70c/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:310b417f2848a83d1437203fcaeea320a74fb7f28af20bf42bf5afc9c01f1c12", size = 2027408, upload-time = "2026-01-27T23:32:46.576Z" }, - { url = "https://files.pythonhosted.org/packages/c6/52/08f98262e77b1cbcc834cc1a5db494d0661ea1dbdea58c2e2d51a57fdaca/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c023539ca6de99234cf5102c3ec0d6af817f5396fc93028a22ba5b834a35b8a", size = 2159245, upload-time = "2026-01-27T23:07:32.664Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1f/751a5a8cfdc95fb4dc556192d37369ae488c30c473fe9a3ec720b23d07ea/nvidia_cudnn_frontend-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:e13f7dd46cdb4762dde87f181f06d1c5e15e9478bbdd547bfa74d9b11f415aae", size = 1591041, upload-time = "2026-01-27T23:09:04.118Z" }, - { url = "https://files.pythonhosted.org/packages/e8/bd/db791a26ebb6a6e1268f518e18c82d8ad18546f7008f4b0d5bde15f927de/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a6e2b7bd43705ffa4af3b187374fdd5e7d09fc228a4d65fc8b4b0a537a8e605", size = 2027249, upload-time = "2026-01-27T23:33:22.46Z" }, - { url = "https://files.pythonhosted.org/packages/19/74/3038cf496d5de7cfdff730f5202e438c17d9123de507059340e02ddff9d7/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0544206b02cae9da4f044ca3fe7416b99e0c8a8052285dd3e5a8fc445d34f9c", size = 2160001, upload-time = "2026-01-27T23:07:50.248Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5e/148cc6609dba326e620e4d949246020dfba05ca07d0387442e62b71d19b6/nvidia_cudnn_frontend-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:7eefa5f10cc003df5f3593f82f1ee6c001fc3412bdc78430c751914dfceefd7f", size = 1591270, upload-time = "2026-01-27T23:09:21.435Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b3/e8c046bfb24663160cbda04903145b2d75b293be94434e23dd7bce1c7419/nvidia_cudnn_frontend-1.26.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b15c3149ace627fa6bced7a68265b107416faf41b8c2261c5b53ae5d5228824", size = 3467061, upload-time = "2026-07-07T20:53:11.867Z" }, + { url = "https://files.pythonhosted.org/packages/91/30/158e5d77616f484f5f11b8b9b9ec18b3a9655d2c52690d63a1056d7061f4/nvidia_cudnn_frontend-1.26.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:719c4e7f2ee2765eeefc144b8a55aac894c306c6e5bf41797373c8f74b30dff6", size = 3618465, upload-time = "2026-07-07T20:53:34.68Z" }, + { url = "https://files.pythonhosted.org/packages/e0/78/5bee0c45a361f1fa4439cc458fb2e7fb01712999141d7553979bf44a3818/nvidia_cudnn_frontend-1.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:e2dcc87fbc5c7e38d0fbf73c982963878bd0fa763a63836d25042f5fd28942bf", size = 2995757, upload-time = "2026-07-07T20:53:55.887Z" }, + { url = "https://files.pythonhosted.org/packages/d8/a3/c74e3743a05d89fbe0ab417f30b54bfb4e246bd34c7bf0be66f594814329/nvidia_cudnn_frontend-1.26.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad20c54dfc042eed8c9ec557e8c6234f73ab66e56174e65fe13e9e41ebc1043e", size = 3468508, upload-time = "2026-07-07T20:54:19.536Z" }, + { url = "https://files.pythonhosted.org/packages/87/a5/d67aca7be7037c8811214df5c299027ac958e6cf53c8f3892e9b0bf5d10d/nvidia_cudnn_frontend-1.26.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2a4981c3825f484187b8ade0203bbca69087b8b7472d1465c809a884f4b2e8", size = 3623719, upload-time = "2026-07-07T20:54:45.621Z" }, + { url = "https://files.pythonhosted.org/packages/2f/79/b1c3419e1893847499dc8e566fa7e40097ae6f5653a41d7a562f82a9df2d/nvidia_cudnn_frontend-1.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:17f86afebb79710a33f7c1c486597b9fb38a46c97e461e3db058c6680403f716", size = 2996800, upload-time = "2026-07-07T20:55:04.283Z" }, + { url = "https://files.pythonhosted.org/packages/19/c4/3f587b73ac2eb6e391aebffb7a7a9ac9ed70e1e0e6a1d90ec0fdcb1a516f/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee50df3468f672aa31402fde4c911ad545d08e1d5e133e15bc55c3679d9fd9ca", size = 3471242, upload-time = "2026-07-07T20:55:30.929Z" }, + { url = "https://files.pythonhosted.org/packages/9b/31/64818fbaa117456349241b42ddfaef3ae6b050f5e99bb0e9d78eb831279b/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a1223c4e2e8bbe6f620148d6848f4eb773dd94bef534d5b748e91232b5be618", size = 3627033, upload-time = "2026-07-07T20:55:47.089Z" }, + { url = "https://files.pythonhosted.org/packages/f9/99/04a37f34b271ed157c6108c3191bd1993ab3fad8d5d66516970bd1e7c12d/nvidia_cudnn_frontend-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f104a40cb6f25cf01b7d7e84cb99af7dedd32b1cb8264384c2f363b7a3f7fb6", size = 2998730, upload-time = "2026-07-07T20:56:09.848Z" }, + { url = "https://files.pythonhosted.org/packages/59/71/b09fa3625b8ab915ef4925e8704bf189754dad8386208fa22304171b81b9/nvidia_cudnn_frontend-1.26.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fee9922c6be2c1b43cb10162e7cd63ce102b0509c0f028e37ac23a959e761b4", size = 3470545, upload-time = "2026-07-07T20:56:30.992Z" }, + { url = "https://files.pythonhosted.org/packages/48/6d/b85a9b36948c0f176a0ea1e137e60e64ea8dc2b2cefc4560b7070536afdc/nvidia_cudnn_frontend-1.26.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf064832e29d74ab5bafa341b1793650496ebfe0c94292ee00b61008ecf9aac4", size = 3626160, upload-time = "2026-07-07T20:56:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7c/32f6a54f1283952c205680f892bd9d5fce111bb949063133a58c7f5cf2c7/nvidia_cudnn_frontend-1.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:067e5bd08a1d25391188eb7206f711d88b916a926b929aec3609af3e43dbae0b", size = 2998579, upload-time = "2026-07-07T20:57:11.771Z" }, ] [[package]] -name = "nvidia-cufft-cu12" -version = "11.3.3.83" +name = "nvidia-cufft" +version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, ] [[package]] -name = "nvidia-cufile-cu12" -version = "1.13.1.3" +name = "nvidia-cufile" +version = "1.15.1.6" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, ] [[package]] -name = "nvidia-curand-cu12" -version = "10.3.9.90" +name = "nvidia-curand" +version = "10.4.0.35" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, ] [[package]] -name = "nvidia-cusolver-cu12" -version = "11.7.3.90" +name = "nvidia-cusolver" +version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, ] [[package]] -name = "nvidia-cusparse-cu12" -version = "12.5.8.93" +name = "nvidia-cusparse" +version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, ] [[package]] -name = "nvidia-cusparselt-cu12" -version = "0.7.1" +name = "nvidia-cusparselt-cu13" +version = "0.8.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, ] [[package]] name = "nvidia-cutlass-dsl" -version = "4.5.0" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "nvidia-cutlass-dsl-libs-cu12" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/71/a3/46fdf77d373b06bc65a0eda6c921c746985fb3e496c90a09be476291ea80/nvidia_cutlass_dsl-4.5.0-py3-none-any.whl", hash = "sha256:3b051fe02ca69422ab840e64d9865667aba288a3984a7ca4ccd038a82aef1344", size = 10178, upload-time = "2026-05-06T01:17:33.592Z" }, + { url = "https://files.pythonhosted.org/packages/8b/1c/fbddb760a0228df87a9e9d1e60b76ecbe6e18035f5853efe0b4563651b2b/nvidia_cutlass_dsl-4.6.0-py3-none-any.whl", hash = "sha256:e3e0e4d8df20d82c8401fa013f4d82021f41daa5fca3d24b55d4a677f2308ca8", size = 10459, upload-time = "2026-07-02T03:23:18.43Z" }, +] + +[package.optional-dependencies] +cu13 = [ + { name = "nvidia-cutlass-dsl-libs-cu13" }, ] [[package]] name = "nvidia-cutlass-dsl-libs-base" -version = "4.5.0" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "cuda-python", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "numpy" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-core" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/3f/7e141bf3c0b878379398d6dd3f20b1d28f6e284744516457234a4d84b46f/nvidia_cutlass_dsl_libs_base-4.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:741452839b4e6b57f5a2e98f6bcdb729d898ee2a3480a3819b0e74a54a274d49", size = 3320004, upload-time = "2026-07-02T03:24:02.642Z" }, + { url = "https://files.pythonhosted.org/packages/20/e8/e64270880d60dd0c04b1fbedf91434d034d0da51b03220a977f691bf5c9a/nvidia_cutlass_dsl_libs_base-4.6.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:481b5837627de419b6b37274c2d97f6060ab9bc61ce7bf19482c12de984da7d5", size = 2824754, upload-time = "2026-07-02T03:24:24.148Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6e/480e2b4c8cfad7271333b44e20e1bcc821632b32b0d5c9c37022ba2e66ee/nvidia_cutlass_dsl_libs_base-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:669200100131a0b8f2876c535ea532c967335936678593c13aced8273ca7523e", size = 3321233, upload-time = "2026-07-02T03:24:44.902Z" }, + { url = "https://files.pythonhosted.org/packages/3c/88/1f259ffe78178e30a90fbddcec49a567aa077929aec2558f80fcec8dd019/nvidia_cutlass_dsl_libs_base-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90a3e7a61d110a8ed005aae83869c6e5dca0723e36298297c0780e21db59c016", size = 2826897, upload-time = "2026-07-02T03:25:06.846Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f8/22653971fcab2a7ed581934f7a2708c9873fa6a8e8eb285422c8eed4ae01/nvidia_cutlass_dsl_libs_base-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6412572899b1c6d182e516b20f2b0a21874ec88d25234e6040fb2a4381de7a1a", size = 3321728, upload-time = "2026-07-02T03:25:28.888Z" }, + { url = "https://files.pythonhosted.org/packages/ce/38/e91f66739d2f8711d1a2457e68cd86d6fbae307ce66ce270a405d4dc6dc7/nvidia_cutlass_dsl_libs_base-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e41cd5db4de4b535c30ae9ca4412b957800a62560019ae91fa51cf3ea89bf254", size = 2824817, upload-time = "2026-07-02T03:25:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5c/0a82b9b2fee054788944d0e7a97b5e63ed0d304969c3b5c4168bed86b71c/nvidia_cutlass_dsl_libs_base-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7b5f5502cc827039f42789e1e2ac9aef7010f4d26ef4b9d66f4ab082da0bcd2c", size = 3321628, upload-time = "2026-07-02T03:26:10.487Z" }, + { url = "https://files.pythonhosted.org/packages/7f/67/6c21b2d140bbd1ad94a2e22a0e2881457f9e363bdff1b35898a2d7d25aa2/nvidia_cutlass_dsl_libs_base-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:7f27357b87c5c797344cca073f1dcf00232aef427daf161adb3cd87b043e37c9", size = 2824911, upload-time = "2026-07-02T03:26:25.033Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-core" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-python" }, + { name = "cuda-python", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "cuda-python", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, { name = "numpy" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "protobuf" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4d/1d0dc5f36f929885417acfff02af94f61d49e6d34acb480c080d4d887555/nvidia_cutlass_dsl_libs_base-4.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c78b18f2b44ca10a91bc76380ebd65bb7b86aa97a9330bae9b73eb0a1bc51d55", size = 75636580, upload-time = "2026-05-06T01:23:19.17Z" }, - { url = "https://files.pythonhosted.org/packages/fe/81/1229637e8a14e1129475b8260a6ce66058148fa85faf10c94f9f95de5ef6/nvidia_cutlass_dsl_libs_base-4.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:5cfdf52bea8feede5e512a094484956693cb87adaafa310991d2876653b1a88e", size = 74505679, upload-time = "2026-05-06T01:26:23.41Z" }, - { url = "https://files.pythonhosted.org/packages/b4/39/155dcbcf942b2c170aa0d1115ef5f2d358d9916ddc7200ab6e70541b97a0/nvidia_cutlass_dsl_libs_base-4.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f8635ad1e0a670323cc729f167067fa880cb56577ec2e79afb80a35ab371912e", size = 75635889, upload-time = "2026-05-06T01:25:20.572Z" }, - { url = "https://files.pythonhosted.org/packages/a4/36/2c2b3fc81a45a1bbbdcfd10c6d9793fd28848e6fefa6d4ed7c7c477f7d2a/nvidia_cutlass_dsl_libs_base-4.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7bb6de91b00a2b392cd834fec174a1461bf0f10a9b6d28086c8f4885aed27218", size = 74505494, upload-time = "2026-05-06T01:27:29.616Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d0/924048cfa43e1cb546735cb332b05a4fb92c63c1a1ac566f06445f9eca58/nvidia_cutlass_dsl_libs_base-4.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f7c133d31fa82ae7db697fd6943a5f9a2c97c8a40ee1056c67ef29fe00974d8", size = 75630723, upload-time = "2026-05-06T01:24:49.842Z" }, - { url = "https://files.pythonhosted.org/packages/c3/8b/2c187400d85f7d2acb328f20499b7b05745dca8485cf6ad247d5f2b434cf/nvidia_cutlass_dsl_libs_base-4.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bd18322d9247f8c033a10ed4e519c4985ca6b4fb578ade382e5a264422ebd915", size = 74505487, upload-time = "2026-05-06T01:26:52.755Z" }, - { url = "https://files.pythonhosted.org/packages/ff/2c/21d5fc62e030a43c0f1a3dab6749fb632026a27d6a60f59975cd29a5d165/nvidia_cutlass_dsl_libs_base-4.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:90a4d802a03963fa36eb287fbc9b40a1374590fc7e8cc1b9673dee8872f75713", size = 75632646, upload-time = "2026-05-06T01:24:19.623Z" }, - { url = "https://files.pythonhosted.org/packages/1c/79/0dca3b465711ffb4c44b4252940cc5f51d2d4905e405707e5c6c2a83d3d6/nvidia_cutlass_dsl_libs_base-4.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8e58b016da5bb09bd1d809d0c025433edb36b279adfbcd107e96361b214bd8bc", size = 74505936, upload-time = "2026-05-06T01:25:52.728Z" }, + { url = "https://files.pythonhosted.org/packages/84/94/e4e2404ac06a477096ccf8127bf5d391510d36cafb4be86c8c15b4873b0d/nvidia_cutlass_dsl_libs_core-4.6.0-py3-none-any.whl", hash = "sha256:f9ea6d313a03cb11fa177da32e8747ad0cac51358850810f36aa6c4736192c27", size = 767713, upload-time = "2026-07-02T03:23:39.876Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-cu12" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "cuda-python", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "numpy" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/11/0bcc6e0b58958f13f5e0f0df8c1f1078ffbacee49b8b3a09f660489aa926/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:27685c36145b23f2c988babe75b4f29b25cd327c6a2189c5ed6edf08f3828508", size = 86991509, upload-time = "2026-07-02T03:28:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c8/82428db415b151c21d4ec3f38f75cfbb88f14056dd0fc684bbfefc94d309/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a1f5b27adab0caa8574e9663c4d2db6063b6d2c4a894489f0e1326c8395f7f7e", size = 88436063, upload-time = "2026-07-02T03:28:35.717Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c4/ea041a7857b3c7e10e939d787feee59c6c8d2d51a9ae1518684d8894daa1/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:783d5da4aa19a4d11429435419b64264c96d429a1794cbc9df2aa905c1342eee", size = 86992109, upload-time = "2026-07-02T03:28:58.136Z" }, + { url = "https://files.pythonhosted.org/packages/1e/98/4501c0b4053cacbb4e555d306d891f2426ce7edbb148f6e78376418e0356/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0479904db0736ab912b2f2db7c6a76cf3c9f6e953f94dc5d667f73c27f18d772", size = 88436391, upload-time = "2026-07-02T03:29:21.26Z" }, + { url = "https://files.pythonhosted.org/packages/11/38/62def848b65bf067f434df7680c7e8c48519b25bbd3f03f9cdff3606353b/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:87f132ccc30946949868989f3b1b1adaa714ccdf5c636e5379b54909cc29576c", size = 86992102, upload-time = "2026-07-02T03:29:41.47Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/f3f8962a9b91dd9368b90e23b2ac81614d6e9df72b55365ec0c216c3f8f9/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:abc341ff0fce40ed0bdadf160f6afac07fb9d01768d4daebd1628c330b3e4210", size = 88436835, upload-time = "2026-07-02T03:30:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/7c/92/773b79f50ca59ca878a5e6be53d7f407deeef56f0b8000bb8cddc2b66d9e/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:45b72e41d343f6b0c1a98669719e03dca4769d7285ae85b7d2a5292168fc73ec", size = 86992268, upload-time = "2026-07-02T03:30:47.112Z" }, + { url = "https://files.pythonhosted.org/packages/07/c1/2521ce3d3f46731d0563bf7e5e0fa6b6ea42c31bcb6763cf4bde16d7b3ce/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:22028842dd9c6064a3de7756b301650be77ddebf6f2acd5336bfbcd05aaf4c02", size = 88437265, upload-time = "2026-07-02T03:31:07.225Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-cu13" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python", version = "12.9.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "cuda-python", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "numpy" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/e4/7e8573b5219d81751fb76e1fc1a94e595248900e9e735e12818e3440aaff/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c41494f927985c089015872278dfa7e8b1281cf35e35f5e587ffc16449e40ff4", size = 86707845, upload-time = "2026-07-02T03:32:59.287Z" }, + { url = "https://files.pythonhosted.org/packages/8f/18/3f05669549141b5b7179eccbf4e951dfeb1b79eb9842ef8740d32f2c05ba/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:4e475bd4729ce3c1a6a660e9230a4c26788d94a718a8609383f417376dde3650", size = 88025224, upload-time = "2026-07-02T03:33:25.043Z" }, + { url = "https://files.pythonhosted.org/packages/15/7c/2b5c8e98511d9f3f778644a31039d3f69763c05a50574a9d31e8a4a3f2ba/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4876ba55f94f2b1d2285be67ec36e39668cf1ac15b5e7db49830ba5897ea9024", size = 86705152, upload-time = "2026-07-02T03:33:54.958Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a44b389a74f67922e11b888b05db7435228a48a69e3d29b3a5ec579ffbdd/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7235c5cfd1db3814bf559d6b976beb759dc7298914f4ff2684a91c3071369598", size = 88026175, upload-time = "2026-07-02T03:34:16.054Z" }, + { url = "https://files.pythonhosted.org/packages/e9/bc/25d542974cc7d2594a22ce1df71c40f8900d44c10aef577cbf5ae7a37e5d/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8c47899a5778dba4f30de76cce5e22bb5dc2a5bde4979ee1196ec9c6468e80e1", size = 86704408, upload-time = "2026-07-02T03:34:36.929Z" }, + { url = "https://files.pythonhosted.org/packages/b1/0f/bd8b25e6307764a7bfefa519241d6e417f3ba1c75ce548aa76b712a4fd15/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4799fabc4bd1f7825ff00101ae0054c5cfb6769aefd6d9694f6da8c0f07e12c3", size = 88026053, upload-time = "2026-07-02T03:34:59.316Z" }, + { url = "https://files.pythonhosted.org/packages/c9/37/089305ba886ae9ce9359ceba7a8289af744efcaf1b44c85cd3d7c803b9a1/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f4c0835344efb02d4058f812f452fe33dc18c7f19eb94268860162500cdd2354", size = 86704416, upload-time = "2026-07-02T03:35:22.412Z" }, + { url = "https://files.pythonhosted.org/packages/59/1d/e39c5b41e5ca59d5a0809fdfb9ac548430ed957a5e806ad85a77d132c15a/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c3fd95c840748879ece42b7ecc9585205e4228ebde55c04ecbce48d2987d905a", size = 88026053, upload-time = "2026-07-02T03:35:48.41Z" }, ] [[package]] @@ -2164,35 +2530,72 @@ wheels = [ ] [[package]] -name = "nvidia-nccl-cu12" -version = "2.27.5" +name = "nvidia-nccl-cu13" +version = "2.28.9" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, ] [[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.8.93" +name = "nvidia-nvjitlink" +version = "13.0.88" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, ] [[package]] -name = "nvidia-nvshmem-cu12" +name = "nvidia-nvshmem-cu13" version = "3.4.5" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:042f2500f24c021db8a06c5eec2539027d57460e1c1a762055a6554f72c369bd", size = 139103095, upload-time = "2025-09-06T00:32:31.266Z" }, + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] [[package]] -name = "nvidia-nvtx-cu12" -version = "12.8.90" +name = "nvidia-nvvm" +version = "13.3.73" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e7/ff646aa6015c7e6d12aad234e68925c87b6681d8d18c3ac40535994a3b0d/nvidia_nvvm-13.3.73-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:0e28e0858a3475e11ac67d35301cd5bf82666a1c0dc4ec4e80ceaf3a5fd1dea8", size = 69250424, upload-time = "2026-06-29T17:08:07.453Z" }, + { url = "https://files.pythonhosted.org/packages/2f/05/35754a7105563fd9b496e5ee8e1acd986aef8258760c3cbccf419aee861a/nvidia_nvvm-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2bcdd5783b5481445f1f0e7170cb836cc0d72999839ba850bbba6dc97b76bb8", size = 66984478, upload-time = "2026-06-29T17:07:43.765Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6b/d5756f485012b920475cbc01457c1b9a7d0485bfb04b92598c5e1ef3e9ab/nvidia_nvvm-13.3.73-py3-none-win_amd64.whl", hash = "sha256:b5c91dfa59ee4cee90b2dfb19c6203f31c914b9c9b5ca10726c2da7cf8ed401d", size = 59981103, upload-time = "2026-06-29T17:21:43.334Z" }, +] + +[[package]] +name = "nvtx" +version = "0.2.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/dd/692765e87de30bae1522cdffaa0f2b52949658a92a0fa6d96b1a01eae9d2/nvtx-0.2.15.tar.gz", hash = "sha256:2287d3be05b85661deb386f878d1f536c2e532774aa9ec7a50c434942ed81ae5", size = 121230, upload-time = "2026-03-18T10:01:25.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/ef/ea1e9d92afd07fdf2a2390e508f1d214e5ba890561d7849d6ca708534b9d/nvtx-0.2.15-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4f50832fd90a1b480a9deef6e4cd48015b61869095b54dd1a7afe87b4138c6a", size = 768543, upload-time = "2026-03-18T10:07:21.819Z" }, + { url = "https://files.pythonhosted.org/packages/32/8e/b42c05cf3cc43c51f21fdda6f7c4fe28a595c6d2bdb0cfbf0477dc5805f2/nvtx-0.2.15-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f3362f0db4252514719326c9d5662b0f93d254659ba97b9c8dbe556286e0e3e", size = 771975, upload-time = "2026-03-18T10:12:23.772Z" }, + { url = "https://files.pythonhosted.org/packages/60/77/fc000055b5bb1651cdd772f0fe1fd9a16c7773b28dfc5624eea331d1415d/nvtx-0.2.15-cp310-cp310-win_amd64.whl", hash = "sha256:d71f934e580d4572f382712b6da464ab69e4c212981506f781f927d5c6d935d6", size = 134503, upload-time = "2026-03-18T10:04:05.773Z" }, + { url = "https://files.pythonhosted.org/packages/80/65/435d10b2041ee082c07d5aed129afd504012c8908796d695f10e66bcc716/nvtx-0.2.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:157b80ea9b4db6c8f47f8dbe2fa2e81e7a7f1445bb87f8268f43dec9210b78a1", size = 806443, upload-time = "2026-03-18T10:05:49.308Z" }, + { url = "https://files.pythonhosted.org/packages/47/bc/be94576ba33af75bcc68a857daade64cb86481764d4fb0f36308b1f6fc85/nvtx-0.2.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02bca69ee55e0be41eabf908de9dbcdd18e702c7f49f9aa63fd396ce684ff5d5", size = 808183, upload-time = "2026-03-18T10:11:16.262Z" }, + { url = "https://files.pythonhosted.org/packages/f6/7a/42109f1cfb1ff9913201cb2b804956a4f003db4c018c2522a3c8066b3a1c/nvtx-0.2.15-cp311-cp311-win_amd64.whl", hash = "sha256:dbe41f78f5a811bd4cdad0a237e5b41a4937d8c2c6c9abdd161091671a598bc0", size = 134631, upload-time = "2026-03-18T10:02:11.247Z" }, + { url = "https://files.pythonhosted.org/packages/c2/07/698355285a03a366ef63ea9762fc1feef3f9f25483e1655408f72d827090/nvtx-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2cc530cd0f1a2c14a3a7e683833db509888ac5ed4ead94e5c9e2c7317c6937a7", size = 807159, upload-time = "2026-03-18T10:09:49.232Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d1/08f22448d83481408d663065764ba583df091a7de629ed38fc97e522f1af/nvtx-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ca8030a6d197952318013dd1c12c22da1d4b9feb76ba72e0fcd449961183c2c", size = 806187, upload-time = "2026-03-18T10:13:32.972Z" }, + { url = "https://files.pythonhosted.org/packages/54/23/c97c39e3b7ba256aa343cb828ca0d1c8421f705ca84795658ecd14ca95ed/nvtx-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:70a1e768964e0520b68ccabc4df391cc227537c45936a7eba6507bc65e617e00", size = 129178, upload-time = "2026-03-18T10:02:55.299Z" }, + { url = "https://files.pythonhosted.org/packages/05/c9/8341224b8284f7deb6a634119939de5885adc421e64b6743693b30da2186/nvtx-0.2.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d28660d9c46f8ba750d781572b6aa5a1e6221abba224ab32d7fb32c2d0fd67df", size = 780787, upload-time = "2026-03-18T10:10:40.634Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c0/4a5bb7897918de7c7e0191d9342df8ae4cb797ff07276e0f20d13e497ce7/nvtx-0.2.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10749686633f880ad53dcdbb2179fad41b45dcf5b7631d4a1070a577577bd386", size = 782575, upload-time = "2026-03-18T10:13:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/38/b9/6b381ac7c5a3ded331aebbf25f8959d19b51d320fb2514c76c6b6edddaaa/nvtx-0.2.15-cp313-cp313-win_amd64.whl", hash = "sha256:a6650b029263d12f8427a4dee8bd59cb9c91bccb60543bfcb20bc2b00fdcd672", size = 128764, upload-time = "2026-03-18T10:02:33.343Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/a9acb6d95d2e0e381b2956544768528dd8d7a9e827af8c2014169d838284/nvtx-0.2.15-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25813ead4fff4d3a6e04f69a72507b096a6bdbecefa369f1100b0e584767bca8", size = 833375, upload-time = "2026-03-18T10:06:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/c7e8645061cc2fc23f3a54f33e1e340df59216f07dcfb97d46b8ae7dd26c/nvtx-0.2.15-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3741edac4678b92f03d22a3f0a2dfd469f422f85e63db71b038e02525b2404ad", size = 788639, upload-time = "2026-03-18T10:12:01.69Z" }, + { url = "https://files.pythonhosted.org/packages/96/03/fadd82acdbca6d1c49ac517081a0c3714346f52f4c7e1d4449d77605b4aa/nvtx-0.2.15-cp313-cp313t-win_amd64.whl", hash = "sha256:8be06c3c8c267eba56a0396366b9593092e0b75ea8d3702b303d48c0a1662f0e", size = 142609, upload-time = "2026-03-18T10:01:48.832Z" }, ] [[package]] @@ -2383,42 +2786,42 @@ wheels = [ [[package]] name = "outlines-core" -version = "0.2.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/d3/e04e9145f8f806723dec9b9e5227ad695a3efcd3ced7794cf7c22b15df5e/outlines_core-0.2.11.tar.gz", hash = "sha256:dfce56f717ff5083e54cbcfdb66cad243365437fccbb5509adaa7e31e030f1d8", size = 197263, upload-time = "2025-05-19T10:12:51.719Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/8f/83c83e2afd142067c7f3cf2e152809195eee72d6a9b6c8745f13b827273d/outlines_core-0.2.11-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:89d79d8454b321f60047541a896d410ca9db631d241960266c4fe839cf5cd1b1", size = 1961650, upload-time = "2025-05-19T10:11:53.12Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e9/c6b99b4364b7026b71badc06b9809a2fc4154d6b0c475bc03ab4471f81e5/outlines_core-0.2.11-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:44d581893f8644da02db7be11887229a40d26077cbdd22072ad1ed1db0ad0b2d", size = 2133920, upload-time = "2025-05-19T10:11:55.15Z" }, - { url = "https://files.pythonhosted.org/packages/f7/b8/cfa2bd8e1260eb1870c42a1a34389e9673a12335d09004ea6f1c82266a5e/outlines_core-0.2.11-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:e88b7f717915d91136d915adb65c2603d2aa6457ec3fc336884bdb0b28d3188a", size = 1960688, upload-time = "2025-05-19T10:11:56.773Z" }, - { url = "https://files.pythonhosted.org/packages/b9/02/4cffd04e360e315b060692bf1a80f84bac1671ef90f12daf765db6d68791/outlines_core-0.2.11-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:8c7ecdba2162e9b30b837251387c26b1a23f80f58d01d02e7600e4b1962c5333", size = 2130263, upload-time = "2025-05-19T10:11:58.1Z" }, - { url = "https://files.pythonhosted.org/packages/4e/85/69a450a486824026eca181a8d573aae3ecfdb25f0c2af852065dde17a372/outlines_core-0.2.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5fcefd221c10c95ce74838869450c6fdbbe2f581f0ba27e57a95232bd88c3a", size = 2289453, upload-time = "2025-05-19T10:11:59.919Z" }, - { url = "https://files.pythonhosted.org/packages/d1/3c/d7cb3eac6870a68b9034854fbfa07e67abfa1fa0d92198b9fee83fe6d044/outlines_core-0.2.11-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a3c7774b112106f3afe931c65637fb3e0725d43707ceff1d34d6899cf0fa8200", size = 2115289, upload-time = "2025-05-19T10:12:01.527Z" }, - { url = "https://files.pythonhosted.org/packages/cc/5f/4cef22e2cf1ec286bd78c0052a0fa7ecf8519144477e7d4e276cbd70c625/outlines_core-0.2.11-cp310-cp310-win32.whl", hash = "sha256:1cfbb4cdcf34be5c6b08d279928b2b1050ed4c5e96e6e8405e3e624305c6799e", size = 1768059, upload-time = "2025-05-19T10:12:03.058Z" }, - { url = "https://files.pythonhosted.org/packages/4a/3a/ce6aceb6545bb1e13cf05c1f34468c5c14c8c8be92cdabcf777b4bb067ef/outlines_core-0.2.11-cp310-cp310-win_amd64.whl", hash = "sha256:670c1c1fca26fb5c7f00dbb11d1f81cca4204863c3dfdeee82017a6846397bf9", size = 2062413, upload-time = "2025-05-19T10:12:05.097Z" }, - { url = "https://files.pythonhosted.org/packages/4d/ca/d5e92e197b40f62deb46dcc55567a51c8bf37943df7bc6658d93f30740f1/outlines_core-0.2.11-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e96b8d0b56afcd3b86f4efca466c578f3725da1148ef62423249c92993841762", size = 1961746, upload-time = "2025-05-19T10:12:06.723Z" }, - { url = "https://files.pythonhosted.org/packages/02/b2/f3d6e7e37ebe1de3c345b53d8dc01e9b5c5f05b20e494fe94bf8972db4b0/outlines_core-0.2.11-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:d108ee8cd5e2fe71c2b0720b949d004901fec8bdb64bcd0c01b8abe38ab7ae1c", size = 2133815, upload-time = "2025-05-19T10:12:07.934Z" }, - { url = "https://files.pythonhosted.org/packages/07/21/62a680da6941b53d765160d22bdcf35849c22b7a987f4e9e8b7db7885c9f/outlines_core-0.2.11-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ebf42ab5b7ae38235d3c3333b5cacd6e91449b87b8a48a85094ea28ad9de9878", size = 1960539, upload-time = "2025-05-19T10:12:09.23Z" }, - { url = "https://files.pythonhosted.org/packages/5f/57/20cfb402aee1a7be0e08d861349570255ad2d17ba7fe7f8fd5706326588c/outlines_core-0.2.11-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:fd4305ff8418d14059d95dc3276ca96ba1b5aa499908e1af8bb3c7207aa7ac68", size = 2129894, upload-time = "2025-05-19T10:12:10.534Z" }, - { url = "https://files.pythonhosted.org/packages/4c/db/32c6e1170f139420e948fdd18a09a6175244bc0760dcf4dc2470e18411b9/outlines_core-0.2.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:132605b8dd1e3d1369da6a851992dd357f6376068292f6bd47caa7a28b794d19", size = 2289078, upload-time = "2025-05-19T10:12:12.118Z" }, - { url = "https://files.pythonhosted.org/packages/25/c3/b6e6f4e08fa84d2424f82705a6dc47fee33cb91989010fa678736957dcf6/outlines_core-0.2.11-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b31d5fc83b78aad282dd667b8d6e684614481fe08a7609ce0ce45dee64cd2991", size = 2115075, upload-time = "2025-05-19T10:12:13.761Z" }, - { url = "https://files.pythonhosted.org/packages/d4/9b/b84c4933e4f35b34e9b23fadd63a365ad8563cc7561d8528b33de4ee8102/outlines_core-0.2.11-cp311-cp311-win32.whl", hash = "sha256:3e316a79f3ecfa12c17746edebcbd66538ee22a43986982f6b96166fb94ee6b1", size = 1768254, upload-time = "2025-05-19T10:12:15.02Z" }, - { url = "https://files.pythonhosted.org/packages/99/5b/380c933c65ca9744c163fe4a3702ad7f3e9ca02e09ac84a09b6837cff9b6/outlines_core-0.2.11-cp311-cp311-win_amd64.whl", hash = "sha256:c260a042b5854ff69291649cfd112066e6bab0dad0bb9cec8a6c3705ef3a59cd", size = 2062167, upload-time = "2025-05-19T10:12:16.443Z" }, - { url = "https://files.pythonhosted.org/packages/5f/2c/c7636823244c70e2960060bf9bd978248dffb55c5e7c91c46d18354b2a24/outlines_core-0.2.11-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4a9db4872bae083631d720994f4cee603bce0536b33d5a988814576863b657cf", size = 1957668, upload-time = "2025-05-19T10:12:18.29Z" }, - { url = "https://files.pythonhosted.org/packages/c7/09/5c62047da139d722317a444a4d01cd5f11943a8c2eaecce784341dd0844a/outlines_core-0.2.11-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:8359a45c59f6a8f2eb717245806501a59044c75f6ea8bd08faaa131cc8cdec45", size = 2130493, upload-time = "2025-05-19T10:12:19.537Z" }, - { url = "https://files.pythonhosted.org/packages/89/7a/d6a2810f90e37d550168e0c0a9a915086ea721444727e3ca2c630898d1ef/outlines_core-0.2.11-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:5d26a46591377340e0b870b8a96ea8341058341a62ee0bded9098e0c88dd24f4", size = 1956804, upload-time = "2025-05-19T10:12:20.755Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ea/339e6c273b5581128c3b7ca27d428d8993c3085912af1a467aa32ef0e9d1/outlines_core-0.2.11-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:ae460a34675fb11d92a5c605a480fbae4cd6c1b2d11b3698da64a7fcaba64dcf", size = 2127085, upload-time = "2025-05-19T10:12:22.02Z" }, - { url = "https://files.pythonhosted.org/packages/92/c7/a65d1fddf49830ebc41422294eacde35286d9f68994a8aa905cb14f5aade/outlines_core-0.2.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86df9740368866295077346440d911df4972da2b3f1f54b8125e6f329e8a8891", size = 2287677, upload-time = "2025-05-19T10:12:24.24Z" }, - { url = "https://files.pythonhosted.org/packages/23/79/8795aed8be9b77dd69d78e7cfbfcf28c179e6b08da6e56bbbf48a09fe55f/outlines_core-0.2.11-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:96ce4dd78f106799be4a0a5795cefd1352806162973756a4b6fce4bb6eddd7e4", size = 2113000, upload-time = "2025-05-19T10:12:25.446Z" }, - { url = "https://files.pythonhosted.org/packages/59/e3/cbe9294b06d92ee1892dbb6f2125d833d68e8629d45d080d6daba54eec2d/outlines_core-0.2.11-cp312-cp312-win32.whl", hash = "sha256:358db161cce3650ba822e118dcf0a1efa571c7deb4864ab9d64ca2c9cca7425d", size = 1765703, upload-time = "2025-05-19T10:12:26.693Z" }, - { url = "https://files.pythonhosted.org/packages/1d/c9/ed3cf362515fac16e313368b9b2f2497051f4ded88679205830b6f889f54/outlines_core-0.2.11-cp312-cp312-win_amd64.whl", hash = "sha256:231f9d20d2630c70665345821780d7808b29539620a75c99f65113b518c51032", size = 2060945, upload-time = "2025-05-19T10:12:28.294Z" }, - { url = "https://files.pythonhosted.org/packages/11/58/df6f57546f7792c990a4380ceaf99243a0b26b24c199e34e0a9277c89976/outlines_core-0.2.11-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0907ff25d79edbf8650268028de85a1b41b38696f147059e007da4626a1031f1", size = 1957172, upload-time = "2025-05-19T10:12:29.737Z" }, - { url = "https://files.pythonhosted.org/packages/9b/cf/b07e33c44544e7865ec481554788807dfa6ad10fd86191ad21f2200f145e/outlines_core-0.2.11-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:f4146da5957f97550eebd19e80635e48035886fd10f03e9735cc111caaf74e93", size = 2130284, upload-time = "2025-05-19T10:12:31.408Z" }, - { url = "https://files.pythonhosted.org/packages/83/70/8f981706e2620914c48fd1edb42f9409d76b84c72149d48e89d14820fab6/outlines_core-0.2.11-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:8776a6db8843187c90e4c54bf94510cda68ca7a11c9b48d90587179fd3224bc2", size = 1956727, upload-time = "2025-05-19T10:12:32.996Z" }, - { url = "https://files.pythonhosted.org/packages/89/de/fba234a9c3984408f017ee0b1ca2e9d6191f8086afa649d3e4b04ed055e2/outlines_core-0.2.11-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:d44f38a89028bed50494420b47d08ebefa78f34b129e2ea6383c801e5ba62c26", size = 2126905, upload-time = "2025-05-19T10:12:34.261Z" }, - { url = "https://files.pythonhosted.org/packages/87/96/7dcdc5198844145ab35528f9f93a58c3d47b87e54d0f79357c631d7b7a9a/outlines_core-0.2.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daef6eaaf8c3403455ab5cbf265cb5c6838df571eb7c4b23cddac19cfc701726", size = 2287320, upload-time = "2025-05-19T10:12:35.515Z" }, - { url = "https://files.pythonhosted.org/packages/4d/68/b420b6a3beaadbf8e9f2a82132120027efd6424634013fbeca8c2fed7467/outlines_core-0.2.11-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:76b2512417c68863f8f227a080e87f755682dfd895e23b021121318be11da579", size = 2112861, upload-time = "2025-05-19T10:12:36.742Z" }, - { url = "https://files.pythonhosted.org/packages/78/d6/7c2a016f7a5eab2f3df2b3a258f270872c78fe0dd7d9fbee87429f1b6b1f/outlines_core-0.2.11-cp313-cp313-win32.whl", hash = "sha256:707eeb3d190485f55a27ad9a6ad70df86688fa2bf405894a118283be7f59bd55", size = 1765574, upload-time = "2025-05-19T10:12:37.98Z" }, - { url = "https://files.pythonhosted.org/packages/a5/39/4c07f1d1f8e6ed85db9fe73a021113795a05aae8a84f36f0bdebb08bfde8/outlines_core-0.2.11-cp313-cp313-win_amd64.whl", hash = "sha256:ad46698564c9b13cbfbc744067de12be73bd740d7b2de20ec6b979ad7511f7c9", size = 2060567, upload-time = "2025-05-19T10:12:39.228Z" }, +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/04/4a0812eb27c086cfd2e66e7ec9150f33e105912a9b7f8b335e3479f03a06/outlines_core-0.2.14.tar.gz", hash = "sha256:64808deed1591ca3029ff64346ceb974cd5d780c916ea82504951fe83523039e", size = 191539, upload-time = "2026-01-09T15:59:10.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d0/e7719044b3ed57fde6f700211be67682eec4a2735fcf8d4199400ac8f7a3/outlines_core-0.2.14-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:056f656ea6e4807338963377afb50b9d936593ba3545a819f1aba56fd6e14920", size = 2050100, upload-time = "2026-01-09T15:58:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ff/f41c933bc5b69b7080e763af375d7632d419859b6cd4492c5a12d5c89c80/outlines_core-0.2.14-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:c76f28feb6ea71b1ff4b0ba5901dc383273a32b156213dc1bc753fc634645a1e", size = 2200812, upload-time = "2026-01-09T15:58:08.876Z" }, + { url = "https://files.pythonhosted.org/packages/ab/3c/05b3e5b9fa4f7f40da459022b2292c6bca48492df920eebd3572844e7fe6/outlines_core-0.2.14-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:e604925d6525f669253160568397df6d6c8124b2e01f1fde553e3b9f28ce9e21", size = 2050299, upload-time = "2026-01-09T15:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ee/abcee1d35c7a7fde984ce4299de2db72add3c8a88a0f6e9e7f8c4836151d/outlines_core-0.2.14-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:f0e5037153b5b3abfb617f6dfdc3ff28b6fab50f0de5936ea6995f5675d23e0b", size = 2197822, upload-time = "2026-01-09T15:58:11.253Z" }, + { url = "https://files.pythonhosted.org/packages/19/cc/6f5d1b92e79b30c2ac187b23ab66b0365e06007a9405b35709d02e94615b/outlines_core-0.2.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e4b5b7c8e50489bea444b095692ddb5d8fb92ea6b949c4a6a3381eca9b691a7", size = 2339071, upload-time = "2026-01-09T15:58:12.488Z" }, + { url = "https://files.pythonhosted.org/packages/66/f7/252c0d4ecc5020d698b23257b9860fbab020c1120c15417ed02a9fd82d19/outlines_core-0.2.14-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b582b5d2f773cff966f37d7a5680d97506792647c93fb2e522283e8a14726e9d", size = 2236310, upload-time = "2026-01-09T15:58:13.942Z" }, + { url = "https://files.pythonhosted.org/packages/b6/60/5599fef9e4184a99684c9cc285081a1b5f4a741b2d6c676466a0ca365d39/outlines_core-0.2.14-cp310-cp310-win32.whl", hash = "sha256:f753edd430ac27e6dcde5a614665888db72b78c666aa160c478afd1eb986fb8b", size = 1841873, upload-time = "2026-01-09T15:58:15.282Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5f/da4daf605ab9c26e854a741c3567eb717e8622d4d17bcd751da5238b039f/outlines_core-0.2.14-cp310-cp310-win_amd64.whl", hash = "sha256:060a0174a6262bfd378763f210e374e52011776849a3a767df9863fb6839c142", size = 2136404, upload-time = "2026-01-09T15:58:16.939Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/a67f0be9546776f71c5df373f38ce6db965abc9845fbcd291b393a20712e/outlines_core-0.2.14-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7770b5e0497e6f4548a8923299d4438d7dd61dc17c2f58acfd5df4d3101bb991", size = 2050098, upload-time = "2026-01-09T15:58:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/34/31/f2e19cc32ea97c1bac4882dbfa693671175a330ad5a735af5b97c2258056/outlines_core-0.2.14-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a2795dc2047821b229457f941a303639e0c14e4c3c5718797540a27b529a062e", size = 2200792, upload-time = "2026-01-09T15:58:19.775Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ea/19e859d4cfcbeceace30ad490f5369c87eab81767238593e20c17f55a390/outlines_core-0.2.14-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:4daa22d677dc6a74c44f9266ec9e3151332dcea4250dd019ea0c75b98ae32938", size = 2050363, upload-time = "2026-01-09T15:58:20.981Z" }, + { url = "https://files.pythonhosted.org/packages/c9/db/188aecb87008ddd293b8d315f26017750a1d7f9e95b8e2756d4a3af08196/outlines_core-0.2.14-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:813b28813b22025c3d079b3b8a20cf5a28c6d5ba29ec21c5b1093442aa5d4e91", size = 2197869, upload-time = "2026-01-09T15:58:22.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/e0be45d4c8ad7d301cdc9917d22ff39211da1e830f92fb07b29c9221b5c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:615566bf8257d2bba8ac192cdfc29d1c4357f57b53672fbd622e821215e4f1bd", size = 2338968, upload-time = "2026-01-09T15:58:23.317Z" }, + { url = "https://files.pythonhosted.org/packages/f2/67/9dab90313460eb250f926e7985d62cebfc33c7580197be8a496de6e9f7c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:81d01cfae29de5671bc5013fd6b2008621157bec3d8be284da7da2dc0672745c", size = 2236169, upload-time = "2026-01-09T15:58:24.575Z" }, + { url = "https://files.pythonhosted.org/packages/ef/91/289996bae3457cf3917ff21e0082e4950cf27a101d0870e16fee94c917e0/outlines_core-0.2.14-cp311-cp311-win32.whl", hash = "sha256:8a5e5f34961fe4d04c389d00f92d624c6318ab3ff00467fbf7c93324458886d9", size = 1841978, upload-time = "2026-01-09T15:58:26.309Z" }, + { url = "https://files.pythonhosted.org/packages/be/65/2d59be2f8c0cca118a6235ab2286615e3c1b2fa9d6768c4ea4b86b556204/outlines_core-0.2.14-cp311-cp311-win_amd64.whl", hash = "sha256:babf97a54662330c55a79fdcab8994f96faa6dcb71b458d4b18c4fb538f5d461", size = 2136353, upload-time = "2026-01-09T15:58:27.443Z" }, + { url = "https://files.pythonhosted.org/packages/66/93/30b9188648a479b32be429a24166db47a7bfdb0f9a8aac4c6dcf569e0a52/outlines_core-0.2.14-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:95e6476d9702d2fcc4e85370dbbfb6933a46c816e9c90107f6ce36eb68b5d64a", size = 2049651, upload-time = "2026-01-09T15:58:28.549Z" }, + { url = "https://files.pythonhosted.org/packages/0d/06/f3557daa8e87d5b95f64de269a301d73ec3c2202ab897c3e1f1cb93eb1db/outlines_core-0.2.14-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:f04731a5e29a190e2cc9f692a1f3fb2414a645355ca7d01b83df43439c38bea8", size = 2201046, upload-time = "2026-01-09T15:58:29.958Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/d8acf778990964c951080d568284e858d466f27dfd6f2674781927faba1c/outlines_core-0.2.14-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:0e4c69f0a8565edb56464c4c9b6c291a10805f3a96dff84182980e90ae1a5e2f", size = 2049558, upload-time = "2026-01-09T15:58:31.003Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/0320b14b49b8379ced1ab195ecf5875dbd2267b90148847541f43bfde6c1/outlines_core-0.2.14-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:63f53cfd9614e754499ae86dd699f3abcecf42d6a4e58d80fd80347881d85960", size = 2197854, upload-time = "2026-01-09T15:58:32.39Z" }, + { url = "https://files.pythonhosted.org/packages/29/29/3a04944407207a5d214879ca5ca33c2bd3e65199a4e927051c1bdaaa4d50/outlines_core-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb2060c240c4507f334965a8948dbeeb22007560d797f6debd92346c0b620cb", size = 2341426, upload-time = "2026-01-09T15:58:33.553Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a7/a77f746272504bac3f628047d56ea1731b61549a3e1d9bbfd226f2968246/outlines_core-0.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1de34681c7e0e7e1551fc9036e4fa3c57986336c905a10536591ceb6d869c258", size = 2236941, upload-time = "2026-01-09T15:58:35.118Z" }, + { url = "https://files.pythonhosted.org/packages/99/0d/9f599d938923ab8ceeff26fdf2f9ea53bea3c962085c4927a08338a32349/outlines_core-0.2.14-cp312-cp312-win32.whl", hash = "sha256:870e8e038853818cb202ccc8cde92251f300f96805bfcc3be1c883adda7b5297", size = 1842940, upload-time = "2026-01-09T15:58:36.544Z" }, + { url = "https://files.pythonhosted.org/packages/f8/df/0f145c52ebd156d80273e2f5278227ea57e0275b2aa863bed33f44f77923/outlines_core-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:87b42440478764cce1353a87d8560ef82f3b39b9d753bfe93195ea3584f369e3", size = 2137266, upload-time = "2026-01-09T15:58:37.831Z" }, + { url = "https://files.pythonhosted.org/packages/13/9d/e6c81c975c123f0639d5f6909c987e510d43e07c2e1e6495b21639c4dec6/outlines_core-0.2.14-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8b3e8d668188282a1f7666732bb8a01958ab134db35bb792e7442a40e55ff1e7", size = 2049297, upload-time = "2026-01-09T15:58:39.184Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d1/5ce55ef724aed0915edc877b6dd610d39b3169e4341154bb53daa022065a/outlines_core-0.2.14-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:66e695b375b180725fb534d9adf298531c152ec3d881e3b9e01c82b5dd269f52", size = 2200944, upload-time = "2026-01-09T15:58:40.257Z" }, + { url = "https://files.pythonhosted.org/packages/32/e3/60ad781251eedcf1496317ecd58eb2e4488717ba63b10494ab49dfd05e5d/outlines_core-0.2.14-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:6bd166d3b07acef2f60d4ede44592a26d3f7d8712876bfc8e22150045def5857", size = 2049607, upload-time = "2026-01-09T15:58:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/bc/2d/662d6a76face5b4b3481f888900d00856c37aa2927341a023866457da212/outlines_core-0.2.14-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:9d45462d7548aa0e17176a691ae73447f3e6bed9658a0cd96fe72eadf7474475", size = 2197755, upload-time = "2026-01-09T15:58:42.861Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9a/4b62903de006d991b58674ff033c1b6fb92be5767360376fc961f6771bdb/outlines_core-0.2.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6453e23f01d98ec48e3a4141d7112792ce77001dfb28d91d6fd89f47009f91ef", size = 2341051, upload-time = "2026-01-09T15:58:44.415Z" }, + { url = "https://files.pythonhosted.org/packages/50/36/1532f7d9ab16c676812d94528e89964aa0d15f12adcb285e6ed86f86f2fe/outlines_core-0.2.14-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7deef6df74cb247f2a3a62f03438ba967456504b0555ec7029f8db834e054448", size = 2236778, upload-time = "2026-01-09T15:58:45.437Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5a/dfd94f15f4c04e691e7fdf30cf8b9b22bf2cbc426b3ef270af3e200596d5/outlines_core-0.2.14-cp313-cp313-win32.whl", hash = "sha256:bb008c7ecc034bcfda0ddc10a4d1f2181a4b61ec1643ee56183dd6fa64139c9d", size = 1842727, upload-time = "2026-01-09T15:58:46.723Z" }, + { url = "https://files.pythonhosted.org/packages/34/35/e24ab5d2116812464380587435297d8ece2f0218c2ba8afc9f541e3a6911/outlines_core-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:eb27e92204b296a063ac58f361153be4e78c8103a96e0b1c085b22d4fc3534cf", size = 2137108, upload-time = "2026-01-09T15:58:47.784Z" }, ] [[package]] @@ -2557,15 +2960,15 @@ wheels = [ [[package]] name = "prometheus-fastapi-instrumentator" -version = "7.1.0" +version = "8.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "prometheus-client" }, { name = "starlette" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/69/6d/24d53033cf93826aa7857699a4450c1c67e5b9c710e925b1ed2b320c04df/prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e", size = 20220, upload-time = "2025-03-19T19:35:05.351Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/f4/cdcebf7094b03b99fba71ac8f56bd6f227973642662f49d272332d8419b3/prometheus_fastapi_instrumentator-8.1.0.tar.gz", hash = "sha256:b77f3043665e8d28e2bbd21017506195a43d9adf1d402d01bf95b494b7e560e1", size = 20492, upload-time = "2026-07-26T11:12:44.202Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/72/0824c18f3bc75810f55dacc2dd933f6ec829771180245ae3cc976195dec0/prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9", size = 19296, upload-time = "2025-03-19T19:35:04.323Z" }, + { url = "https://files.pythonhosted.org/packages/44/b9/91a2246e6cf01b7ccb14479803c8a50f9c258ae5c6a0f16ba3294820632b/prometheus_fastapi_instrumentator-8.1.0-py3-none-any.whl", hash = "sha256:b9f40b2cff3f7891ca0610b3ae4fc6ec723fd326b04bb659819aaeb821a0fc7d", size = 19649, upload-time = "2026-07-26T11:12:45.168Z" }, ] [[package]] @@ -2994,6 +3397,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, ] +[[package]] +name = "pyelftools" +version = "0.33" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/11/767522582afab1b884d277de0e6e011640cb9d7292a38694b4b1a1df1ae8/pyelftools-0.33.tar.gz", hash = "sha256:660d82dcbeb8e83d1702bd97f223f761625da06111c0cc988eac6b8ab0c1b61f", size = 15068655, upload-time = "2026-05-29T12:56:22.553Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/2a/f9697576603dae937727827505a6126a066affb227034e77e6f9068910da/pyelftools-0.33-py3-none-any.whl", hash = "sha256:f215ad5f47d3f1373a21496a6c9e0707c622840d0622f23ff7ce08678b020036", size = 201178, upload-time = "2026-05-29T12:56:20.587Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -3020,6 +3432,28 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pynvvideocodec" +version = "2.0.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/9d/bcde031d2b7ee86336459236729dbb7a8a58d6fef0b0d78f99fe8929db13/pynvvideocodec-2.0.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c7a133a8088ff9152f7953ff63c5cbc27801f0313d1e577c5f3e65da67abf570", size = 28632415, upload-time = "2026-07-08T04:25:07.008Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a9/b80e3b2cb324f4fa684f66748f1fa1f14e8e1687309408cf3e1e8c1c2a81/pynvvideocodec-2.0.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7af1d9ab37f7415168c23be00539094e7b21774661af2a0a2b9bb574670bc75e", size = 43180627, upload-time = "2026-05-27T04:00:41.809Z" }, + { url = "https://files.pythonhosted.org/packages/92/d7/a57b8672ea2fb5edfb53a61b2478859d52c593d40c1e061b558c31a743c4/pynvvideocodec-2.0.4-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:8d6d3d63317c452bdd18167f622abb392bd982ae65293673a169e3f253c1d1ed", size = 35754785, upload-time = "2026-05-27T04:01:15.377Z" }, + { url = "https://files.pythonhosted.org/packages/72/bf/e181180f9e9a60dac64640f8eeaf5635184c6aa11663f7855339c440f3a5/pynvvideocodec-2.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:f678e5a6903498f291cffaba5d47b023c67bf18172049186044c4466492ad443", size = 25687790, upload-time = "2026-05-27T04:02:05.621Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ce/4559ca81f39b14cc121ed284afc017fe36c3aa40e72ef12170b75d7d2113/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4dba42331f6d319087d05359787c7c542483fab22107c07b983cad928c1e3cfe", size = 28632422, upload-time = "2026-07-08T04:25:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/2e/86/8766b11b0884fe9c0530870e99c68af1ce68568ebe0652b636caaa9ea50a/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:bd3779ff73ad703393c0a19c3650f269ca25e71902c24efa0719ed4a58cd9390", size = 43180651, upload-time = "2026-05-27T04:02:38.117Z" }, + { url = "https://files.pythonhosted.org/packages/a9/aa/dd826d41581aa6b11a4c922752a89a0dfbfe5f9095de11f8bb10bde46c1d/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:f809fb18929ac2af042835f10c7679b1c86db9817776f22bc7467907c5c3d918", size = 35755024, upload-time = "2026-05-27T04:03:02.004Z" }, + { url = "https://files.pythonhosted.org/packages/cc/df/c5b516bb16c42c312d3564999f25487607ed105f656ef7fbeb6b13ff3c6e/pynvvideocodec-2.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:fc299a14e61832850be91f8441669ada6fc270903ad1c50acde8e9353590fd3a", size = 25688228, upload-time = "2026-05-27T04:03:29.487Z" }, + { url = "https://files.pythonhosted.org/packages/38/1c/78f6fdf85133157a6a3405eab5ef4c2bc8048194dbda1c91bb9b8645bb36/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bad9e25f494abdcfa8f9dffa33a840509eda3ffcdf6e7cf6465d73be307c0c82", size = 28630316, upload-time = "2026-07-08T04:25:54.596Z" }, + { url = "https://files.pythonhosted.org/packages/ca/49/98da271686e00676f41b1197ba5431ddc341b96d8efb68ea9d68e2b0d870/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b59cec7a1a3f78fad13fead78cad8b6d9686827f9ff4477080245457675a01d0", size = 43176147, upload-time = "2026-05-27T04:04:08.297Z" }, + { url = "https://files.pythonhosted.org/packages/42/80/7b13c12fd5f3243b01190130ce098a44ddf62e030e6ed712911cbfe40311/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:a0daa28b09705806c8c6b26326df217c45e60c0a12a673ea3ea6ee5e2e7193b0", size = 35754893, upload-time = "2026-05-27T04:04:43.866Z" }, + { url = "https://files.pythonhosted.org/packages/76/50/eb1571ab1cee8ebb8a7bdfc355078beebe4b2bb2e5c6ad5d0e18ab8585db/pynvvideocodec-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:46e2adb82dc6ac333d3535cc76e4e25c7e8d80dd272b1aba0c28702b861d5261", size = 25692590, upload-time = "2026-05-27T04:05:17.164Z" }, + { url = "https://files.pythonhosted.org/packages/46/d6/8475720d4f0f8fc9e852e0092b308643b71f24d9495ba3bd03c38b16c28d/pynvvideocodec-2.0.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:fcb06ef6ef24ee33b8f34950696a4e01636f2d8cdb96c407cd692931d049ac3d", size = 43176124, upload-time = "2026-05-27T04:06:26.415Z" }, + { url = "https://files.pythonhosted.org/packages/3d/28/33d2a4d69b48823801c0b02b1bfbeac04cd9d429ee698d248e7ba946c516/pynvvideocodec-2.0.4-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:51724c6a0e3623c092cccdf93c8b09cced6881f3d0c76653f6ffbec0371f29cd", size = 35754919, upload-time = "2026-05-27T04:07:09.962Z" }, + { url = "https://files.pythonhosted.org/packages/49/8e/8c08bbffc9021db762b6ac103ce544db4e517f9816118a35db9f0c7d2a9a/pynvvideocodec-2.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:8e704a2b553a35cc2de10543a232e7feddc473f3e5478996595236e3194efc5d", size = 25692569, upload-time = "2026-05-27T04:07:38.663Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -3208,7 +3642,7 @@ wheels = [ [[package]] name = "quack-kernels" -version = "0.4.1" +version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -3217,9 +3651,9 @@ dependencies = [ { name = "torch" }, { name = "torch-c-dlpack-ext" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/58/58b82e91b236539f424ff5681e7095b1f2860ddfb7778fe0be14d8fb58de/quack_kernels-0.4.1.tar.gz", hash = "sha256:9d7d6ba412bc0c8a9b1331c52a73db76280adb9dc2f2750df4851ddabef1466b", size = 274766, upload-time = "2026-04-30T14:37:55.65Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/17/890875f88f4d7da28faec9e6cf0a0cc565715b01474e50501fafc5bc71b4/quack_kernels-0.6.1.tar.gz", hash = "sha256:a694f89c91d137478de523c0227365a331ac9cb66790cfb08baa3dbfaafc71e7", size = 387353, upload-time = "2026-07-05T11:50:19.807Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/e4/a6c3bbbe3d4242fa412454b8e8069a079e500be331aecf8f2aa666164e9c/quack_kernels-0.4.1-py3-none-any.whl", hash = "sha256:c1c8df2935bf5156ec47d2c5384ac08b411fd0ee702d80ae916dbf6d6f5ae813", size = 260827, upload-time = "2026-04-30T14:37:54.584Z" }, + { url = "https://files.pythonhosted.org/packages/2b/65/a38a30a6ac96a757363a5be9d09cef799640bb143a64ba5a2f4d400d95d9/quack_kernels-0.6.1-py3-none-any.whl", hash = "sha256:266705ea82117e9b1c8a9e44d68a458519f2498d966c0efffd6812120c3995ad", size = 358439, upload-time = "2026-07-05T11:50:18.502Z" }, ] [[package]] @@ -3794,15 +4228,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.52.1" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/68/79977123bb7be889ad680d79a40f339082c1978b5cfcf62c2d8d196873ac/starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933", size = 2653702, upload-time = "2026-01-18T13:34:11.062Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] @@ -3882,6 +4316,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, ] +[[package]] +name = "tilelang" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "cloudpickle" }, + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "psutil" }, + { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "torch" }, + { name = "torch-c-dlpack-ext" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "z3-solver" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/70/5051f65821baa30a3d61fc48f8ba10c776490315e8c90f82559b92089756/tilelang-0.1.9.tar.gz", hash = "sha256:287f727c913bb648fcf6c1968809ba3390e55eeed257a5c6bb9a80bc05966af4", size = 93395292, upload-time = "2026-04-22T09:19:11.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/db/4dd76da8c8585c605639a21bc098d504e317fe324a72f01ce3c7370250b4/tilelang-0.1.9-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:00ed594fdeb229c5505b9ffa895c3c5daeb28641c78f783fa1f724cf1e08cecd", size = 36599020, upload-time = "2026-04-22T09:14:39.366Z" }, + { url = "https://files.pythonhosted.org/packages/f7/8a/1cbeee79d62abaa02441c2d00621554e41aa62dbf3b94a4feb3867184b01/tilelang-0.1.9-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bbccfe9035aed775ffafb6dc25a5994504b24e2c5d95d0f39643edfafa7bf12", size = 45419374, upload-time = "2026-04-22T09:15:56.014Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a7/f4bfb86f87e107703146e703204cec2c0eae2492b633e0052b0ace3febb6/tilelang-0.1.9-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:77ab0ee2f40f66ea015b6b21426d482751e28cbc635ef9d1198cbd6502454a7c", size = 42110365, upload-time = "2026-04-22T09:17:18.292Z" }, +] + [[package]] name = "tokenizers" version = "0.22.2" @@ -3912,6 +4370,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/8c/b1c87148aa15e099243ec9f0cf9d0e970cc2234c3257d558c25a2c5304e6/tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5", size = 3373542, upload-time = "2026-01-05T10:40:52.803Z" }, ] +[[package]] +name = "tokenspeed-mla" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cutlass-dsl", marker = "sys_platform != 'darwin'" }, + { name = "tokenspeed-triton", marker = "sys_platform != 'darwin'" }, + { name = "torch", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/65/81d7e9f14472bc4c6abb576c9b1edd8e40ab01832027c4e647bfe2890749/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:952209cf4b29a54e6b6e7e088be9d4a40f24792b06fdfa330de8077b9926a7d9", size = 752341, upload-time = "2026-06-24T03:36:28.619Z" }, + { url = "https://files.pythonhosted.org/packages/27/df/0037ade72b165ac97859040919e006aa3d80cb8cc3a79420fb6c03eb16a0/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6a7526d7327746893f8c20d24aa63ba5b8a123d0dfd6e66388e13b768b6452c6", size = 755827, upload-time = "2026-06-24T03:36:29.96Z" }, +] + +[[package]] +name = "tokenspeed-triton" +version = "3.8.10.post20260721" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9e/4d659040d930ced49a7ce663308e7f70eae0b22a4c6fa62e0fdc0b127885/tokenspeed_triton-3.8.10.post20260721-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0a5fb817db3879a645e34430834cf25d9cf4df7bbf9ea9921eb0053d9aa292d", size = 82961227, upload-time = "2026-07-21T17:14:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c6/c402ef70ebf915e89c479fe61c84ba0018a0da89b3727bf9c0fceb2ec54f/tokenspeed_triton-3.8.10.post20260721-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d5462b6bf6f64b0b4702995a9488ba63d6c544eb7fdc64d31f465f12eef61be", size = 87206781, upload-time = "2026-07-21T17:14:27.689Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ad/748a3b1b3e6559e3008c2dbfa4160c4356e5ff80d427e8eb25881bf42873/tokenspeed_triton-3.8.10.post20260721-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0868b11dc177a44e37cf052f4cf00b55a503672d6bf559689b71f58ebb1fd19d", size = 82963642, upload-time = "2026-07-21T17:14:31.324Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/f96a19c3c09a9f39a3f86eeb7f64aeed3793bdfa033e406a5f5b1a90b606/tokenspeed_triton-3.8.10.post20260721-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b66e0587b925e4bfacaf56dd1edbaa149fd6456880387287bb8cda8cdadfbb", size = 87207488, upload-time = "2026-07-21T17:14:34.953Z" }, + { url = "https://files.pythonhosted.org/packages/1d/44/89740db8951918c9acd8731243eef8b44d0eb92ea423552639265c46018e/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d793ad0eaebb1d08272c97a2b8f2c31304231748b03de9a08e70a362de92a6e0", size = 82966664, upload-time = "2026-07-21T17:14:38.568Z" }, + { url = "https://files.pythonhosted.org/packages/91/53/f46b401e8ec8998f5b9c39cff0614b796bf49113a09f588cfdfa342789a3/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66cba8d32a1539afd0ff3eec1782b082d01b4db6824d68017d1d789a03d0be37", size = 87210295, upload-time = "2026-07-21T17:14:42.173Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -3950,65 +4436,46 @@ wheels = [ [[package]] name = "torch" -version = "2.10.0" +version = "2.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cuda-bindings", version = "13.3.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, { name = "filelock" }, { name = "fsspec" }, { name = "jinja2" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, { name = "sympy" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "sys_platform == 'linux'" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:2b980edd8d7c0a68c4e951ee1856334a43193f98730d97408fbd148c1a933313", size = 79411457, upload-time = "2026-02-10T21:44:59.189Z" }, - { url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" }, - { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, - { url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" }, - { url = "https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321", size = 915510070, upload-time = "2026-03-11T14:16:39.386Z" }, - { url = "https://files.pythonhosted.org/packages/36/ab/7b562f1808d3f65414cd80a4f7d4bb00979d9355616c034c171249e1a303/torch-2.10.0-3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac5bdcbb074384c66fa160c15b1ead77839e3fe7ed117d667249afce0acabfac", size = 915518691, upload-time = "2026-03-11T14:15:43.147Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" }, - { url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" }, - { url = "https://files.pythonhosted.org/packages/0c/1a/c61f36cfd446170ec27b3a4984f072fd06dab6b5d7ce27e11adb35d6c838/torch-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5276fa790a666ee8becaffff8acb711922252521b28fbce5db7db5cf9cb2026d", size = 145992962, upload-time = "2026-01-21T16:24:14.04Z" }, - { url = "https://files.pythonhosted.org/packages/b5/60/6662535354191e2d1555296045b63e4279e5a9dbad49acf55a5d38655a39/torch-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aaf663927bcd490ae971469a624c322202a2a1e68936eb952535ca4cd3b90444", size = 915599237, upload-time = "2026-01-21T16:23:25.497Z" }, - { url = "https://files.pythonhosted.org/packages/40/b8/66bbe96f0d79be2b5c697b2e0b187ed792a15c6c4b8904613454651db848/torch-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:a4be6a2a190b32ff5c8002a0977a25ea60e64f7ba46b1be37093c141d9c49aeb", size = 113720931, upload-time = "2026-01-21T16:24:23.743Z" }, - { url = "https://files.pythonhosted.org/packages/76/bb/d820f90e69cda6c8169b32a0c6a3ab7b17bf7990b8f2c680077c24a3c14c/torch-2.10.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:35e407430795c8d3edb07a1d711c41cc1f9eaddc8b2f1cc0a165a6767a8fb73d", size = 79411450, upload-time = "2026-01-21T16:25:30.692Z" }, - { url = "https://files.pythonhosted.org/packages/78/89/f5554b13ebd71e05c0b002f95148033e730d3f7067f67423026cc9c69410/torch-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3282d9febd1e4e476630a099692b44fdc214ee9bf8ee5377732d9d9dfe5712e4", size = 145992610, upload-time = "2026-01-21T16:25:26.327Z" }, - { url = "https://files.pythonhosted.org/packages/ae/30/a3a2120621bf9c17779b169fc17e3dc29b230c29d0f8222f499f5e159aa8/torch-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2f9edd8dbc99f62bc4dfb78af7bf89499bca3d753423ac1b4e06592e467b763", size = 915607863, upload-time = "2026-01-21T16:25:06.696Z" }, - { url = "https://files.pythonhosted.org/packages/6f/3d/c87b33c5f260a2a8ad68da7147e105f05868c281c63d65ed85aa4da98c66/torch-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:29b7009dba4b7a1c960260fc8ac85022c784250af43af9fb0ebafc9883782ebd", size = 113723116, upload-time = "2026-01-21T16:25:21.916Z" }, - { url = "https://files.pythonhosted.org/packages/61/d8/15b9d9d3a6b0c01b883787bd056acbe5cc321090d4b216d3ea89a8fcfdf3/torch-2.10.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:b7bd80f3477b830dd166c707c5b0b82a898e7b16f59a7d9d42778dd058272e8b", size = 79423461, upload-time = "2026-01-21T16:24:50.266Z" }, - { url = "https://files.pythonhosted.org/packages/cc/af/758e242e9102e9988969b5e621d41f36b8f258bb4a099109b7a4b4b50ea4/torch-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5fd4117d89ffd47e3dcc71e71a22efac24828ad781c7e46aaaf56bf7f2796acf", size = 145996088, upload-time = "2026-01-21T16:24:44.171Z" }, - { url = "https://files.pythonhosted.org/packages/23/8e/3c74db5e53bff7ed9e34c8123e6a8bfef718b2450c35eefab85bb4a7e270/torch-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:787124e7db3b379d4f1ed54dd12ae7c741c16a4d29b49c0226a89bea50923ffb", size = 915711952, upload-time = "2026-01-21T16:23:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/6e/01/624c4324ca01f66ae4c7cd1b74eb16fb52596dce66dbe51eff95ef9e7a4c/torch-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c66c61f44c5f903046cc696d088e21062644cbe541c7f1c4eaae88b2ad23547", size = 113757972, upload-time = "2026-01-21T16:24:39.516Z" }, - { url = "https://files.pythonhosted.org/packages/c9/5c/dee910b87c4d5c0fcb41b50839ae04df87c1cfc663cf1b5fca7ea565eeaa/torch-2.10.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:6d3707a61863d1c4d6ebba7be4ca320f42b869ee657e9b2c21c736bf17000294", size = 79498198, upload-time = "2026-01-21T16:24:34.704Z" }, - { url = "https://files.pythonhosted.org/packages/c9/6f/f2e91e34e3fcba2e3fc8d8f74e7d6c22e74e480bbd1db7bc8900fdf3e95c/torch-2.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5c4d217b14741e40776dd7074d9006fd28b8a97ef5654db959d8635b2fe5f29b", size = 146004247, upload-time = "2026-01-21T16:24:29.335Z" }, - { url = "https://files.pythonhosted.org/packages/98/fb/5160261aeb5e1ee12ee95fe599d0541f7c976c3701d607d8fc29e623229f/torch-2.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6b71486353fce0f9714ca0c9ef1c850a2ae766b409808acd58e9678a3edb7738", size = 915716445, upload-time = "2026-01-21T16:22:45.353Z" }, - { url = "https://files.pythonhosted.org/packages/6a/16/502fb1b41e6d868e8deb5b0e3ae926bbb36dab8ceb0d1b769b266ad7b0c3/torch-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2ee399c644dc92ef7bc0d4f7e74b5360c37cdbe7c5ba11318dda49ffac2bc57", size = 113757050, upload-time = "2026-01-21T16:24:19.204Z" }, - { url = "https://files.pythonhosted.org/packages/1a/0b/39929b148f4824bc3ad6f9f72a29d4ad865bcf7ebfc2fa67584773e083d2/torch-2.10.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:3202429f58309b9fa96a614885eace4b7995729f44beb54d3e4a47773649d382", size = 79851305, upload-time = "2026-01-21T16:24:09.209Z" }, - { url = "https://files.pythonhosted.org/packages/d8/14/21fbce63bc452381ba5f74a2c0a959fdf5ad5803ccc0c654e752e0dbe91a/torch-2.10.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:aae1b29cd68e50a9397f5ee897b9c24742e9e306f88a807a27d617f07adb3bd8", size = 146005472, upload-time = "2026-01-21T16:22:29.022Z" }, - { url = "https://files.pythonhosted.org/packages/54/fd/b207d1c525cb570ef47f3e9f836b154685011fce11a2f444ba8a4084d042/torch-2.10.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6021db85958db2f07ec94e1bc77212721ba4920c12a18dc552d2ae36a3eb163f", size = 915612644, upload-time = "2026-01-21T16:21:47.019Z" }, - { url = "https://files.pythonhosted.org/packages/36/53/0197f868c75f1050b199fe58f9bf3bf3aecac9b4e85cc9c964383d745403/torch-2.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ff43db38af76fda183156153983c9a096fc4c78d0cd1e07b14a2314c7f01c2c8", size = 113997015, upload-time = "2026-01-21T16:23:00.767Z" }, - { url = "https://files.pythonhosted.org/packages/0e/13/e76b4d9c160e89fff48bf16b449ea324bda84745d2ab30294c37c2434c0d/torch-2.10.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:cdf2a523d699b70d613243211ecaac14fe9c5df8a0b0a9c02add60fb2a413e0f", size = 79498248, upload-time = "2026-01-21T16:23:09.315Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f2/c1690994afe461aae2d0cac62251e6802a703dec0a6c549c02ecd0de92a9/torch-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c0d7fcfbc0c4e8bb5ebc3907cbc0c6a0da1b8f82b1fc6e14e914fa0b9baf74e", size = 80526521, upload-time = "2026-03-23T18:12:06.86Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f0/98ae802fa8c09d3149b0c8690741f3f5753c90e779bd28c9613257295945/torch-2.11.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4cf8687f4aec3900f748d553483ef40e0ac38411c3c48d0a86a438f6d7a99b18", size = 419723025, upload-time = "2026-03-23T18:11:43.774Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1e/18a9b10b4bd34f12d4e561c52b0ae7158707b8193c6cfc0aad2b48167090/torch-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1b32ceda909818a03b112006709b02be1877240c31750a8d9c6b7bf5f2d8a6e5", size = 530589207, upload-time = "2026-03-23T18:11:23.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/40/2d532e8c0e23705be9d1debce5bc37b68d59a39bda7584c26fe9668076fe/torch-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3c712ae6fb8e7a949051a953fc412fe0a6940337336c3b6f905e905dac5157f", size = 114518313, upload-time = "2026-03-23T18:11:58.281Z" }, + { url = "https://files.pythonhosted.org/packages/ae/0d/98b410492609e34a155fa8b121b55c7dca229f39636851c3a9ec20edea21/torch-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b6a60d48062809f58595509c524b88e6ddec3ebe25833d6462eeab81e5f2ce4", size = 80529712, upload-time = "2026-03-23T18:12:02.608Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bd/9912d30b68845256aabbb4a40aeefeef3c3b20db5211ccda653544ada4b6/torch-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:73e24aaf8f36ab90d95cd1761208b2eb70841c2a9ca1a3f9061b39fc5331b708", size = 114519675, upload-time = "2026-03-23T18:11:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, + { url = "https://files.pythonhosted.org/packages/87/89/5ea6722763acee56b045435fb84258db7375c48165ec8be7880ab2b281c5/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6debd97ccd3205bbb37eb806a9d8219e1139d15419982c09e23ef7d4369d18", size = 80606801, upload-time = "2026-03-23T18:10:18.649Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/3e3fcdd388fbe54e29fd3f991f36846ff4ac90b0d0181e9c8f7236565f82/torch-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:4dda3b3f52d121063a731ddb835f010dc137b920d7fec2778e52f60d8e4bf0cd", size = 114555842, upload-time = "2026-03-23T18:09:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/db/38/8ac78069621b8c2b4979c2f96dc8409ef5e9c4189f6aac629189a78677ca/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8b394322f49af4362d4f80e424bcaca7efcd049619af03a4cf4501520bdf0fb4", size = 80959574, upload-time = "2026-03-23T18:10:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, + { url = "https://files.pythonhosted.org/packages/48/6b/30d1459fa7e4b67e9e3fe1685ca1d8bb4ce7c62ef436c3a615963c6c866c/torch-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a97b94bbf62992949b4730c6cd2cc9aee7b335921ee8dc207d930f2ed09ae2db", size = 114793702, upload-time = "2026-03-23T18:09:47.304Z" }, ] [[package]] @@ -4040,37 +4507,57 @@ wheels = [ [[package]] name = "torchaudio" -version = "2.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/59/88ab8ebff9d91f1f1365088b30f1b9ccce07c5eeac666038a5dee5e2f9b1/torchaudio-2.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cde383582a6240c1315443df5c5638863e96b03acf1cb44a298aff07a72d373", size = 734944, upload-time = "2026-01-21T16:28:49.535Z" }, - { url = "https://files.pythonhosted.org/packages/9b/d6/41f25f9ae9b37c191bed4cd474e403626685d2be8f7d20d011e6601fede1/torchaudio-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:cfb2ad4b7847d81931989127d803487263c8284f21156e9000daec1ac16c0831", size = 390449, upload-time = "2026-01-21T16:28:48.585Z" }, - { url = "https://files.pythonhosted.org/packages/43/ac/a14425fddd1cf56bb052a3bfd38880258008f8c3cd17f37bba55b3a88ce7/torchaudio-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:316cdb15fb37290fca89894b095d97b4dc14a90c4c61148ae5c96bb334d962cd", size = 1891070, upload-time = "2026-01-21T16:28:47.323Z" }, - { url = "https://files.pythonhosted.org/packages/6e/03/d1898db1bf7ecd47ca9b4e1b70927597d236cf721e3736d953d555901832/torchaudio-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:013079d1ba2a652184703e671b8339cbc7991f17e4ed927071fe7635f908a4a1", size = 474045, upload-time = "2026-01-21T16:28:46.191Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e7/401fe1d024bf9352371d854be6f339ad9928669e6bc8a5ba08e9dbce81cf/torchaudio-2.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bcab0e39eb18da84cba1a0c87f600abb6ce97c882200cb46e841caea106f037f", size = 736373, upload-time = "2026-01-21T16:28:41.589Z" }, - { url = "https://files.pythonhosted.org/packages/6f/b7/c66dc34a27441d78997e20d0ffe2f5ad73db9f7b1267511be255bb94ac9b/torchaudio-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:87c841a21e82703ebd4a29170c4e60c25a2b47312dc212930087ad58965ac0c8", size = 391843, upload-time = "2026-01-21T16:28:43.093Z" }, - { url = "https://files.pythonhosted.org/packages/13/ae/a2a34a64947c4fa4a61b4c86d8f36fbcb4ebfec30fdde140267db260f96c/torchaudio-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b2c77fb9114dd463dc805560bf55a1ac2a52e219794cc32b7b32cf2aeffd2826", size = 1894140, upload-time = "2026-01-21T16:28:35.892Z" }, - { url = "https://files.pythonhosted.org/packages/69/26/cd2aec609b4f8918e4e85e5c6a3f569bc7b5f72a7ecba3f784077102749c/torchaudio-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:4c6e9609046143b30a30183893d23ff1ce5de603dbe914b3cce5cc29f5aa5a9c", size = 474792, upload-time = "2026-01-21T16:28:45.254Z" }, - { url = "https://files.pythonhosted.org/packages/0f/36/28a6f3e857616cf7576bdbf8170e483b8c5d0a1f8d349ecb2b75921236aa/torchaudio-2.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d0fbdbfd2f621c51d28571050d6d0c7287791034e5c7303b31480af1258f33f", size = 737144, upload-time = "2026-01-21T16:28:44.189Z" }, - { url = "https://files.pythonhosted.org/packages/ea/3f/df620439a76ece170472d41438d11a1545d5db5dc9f1eaeab8c6e055a328/torchaudio-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:42b148a0921a3721abd1f6ae098b1ec9f89703e555c4f7a0d44da87b8decbcb9", size = 391973, upload-time = "2026-01-21T16:28:39.732Z" }, - { url = "https://files.pythonhosted.org/packages/98/25/e55a30d7138f8fe56ed006df25b0a3c27681f0ec7bc9989e1778e6d559c3/torchaudio-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0e77b2956448d63790a99beed0b74ac8b8cd3a94dcdd9ad01974411078f46278", size = 1895234, upload-time = "2026-01-21T16:28:37.034Z" }, - { url = "https://files.pythonhosted.org/packages/be/a0/da53c7d20fac15f66f8838653b91162de1bf21fb40fee88cf839e4ef5174/torchaudio-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f76a01ecebf1869e1f2c50a261f1cf07e5fccb24402b4e9bbb82d6725b9c7dd", size = 475470, upload-time = "2026-01-21T16:28:40.615Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/341e7bd588355f82c5180103cb2f8070a72ab1be920ab27553a1135d4aa6/torchaudio-2.10.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8fd38d28ee150c584d3ee3b05f39e021f0ad8a8ec8fec1f26dfe150c9db9b2f5", size = 737164, upload-time = "2026-01-21T16:28:38.354Z" }, - { url = "https://files.pythonhosted.org/packages/49/fd/831c2595c81b17141180ca11ab3c0836cc544ef13e15aa0e7b2cb619e582/torchaudio-2.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5bc39ff3ea341097ce1ab023dd88c9dd8ca5f96ebf48821e7d23766137bb55d7", size = 392757, upload-time = "2026-01-21T16:28:33.631Z" }, - { url = "https://files.pythonhosted.org/packages/8e/d8/405c80c57dc68ca5855bddfaae57c3d84ea7397bf1eb2aa5d59c9fa1d3a9/torchaudio-2.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3057c4286db5673d266124a2a10ca54e19f516772e9057f44573a7da5b85e328", size = 1897099, upload-time = "2026-01-21T16:28:24.793Z" }, - { url = "https://files.pythonhosted.org/packages/73/cf/0e48d67788c935e3b3d00e6f55a930a54a67f432e04c33ef80a38cb764fd/torchaudio-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:99e74d1901742bc10961d807fe75c0dd9496f4a4a4ff4bb317c5de4a0b6f24e6", size = 475476, upload-time = "2026-01-21T16:28:28.249Z" }, - { url = "https://files.pythonhosted.org/packages/48/29/30bcce0f17a8279b051b09250993691a828f89a03278306b23571c18df04/torchaudio-2.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6cfe98ef0ea9bee6d6297493ce67ce0c54a38d80caf6535a3ae48900fd5f3769", size = 742449, upload-time = "2026-01-21T16:28:29.556Z" }, - { url = "https://files.pythonhosted.org/packages/43/8c/653e7f67855424bf3b7cbb48335f8316f7fb02bb01a6cab38f6bf9555676/torchaudio-2.10.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:b41b254d958632dc00dc7768431cadda516c91641d798775cbb19bcd4f0d2be4", size = 393430, upload-time = "2026-01-21T16:28:34.855Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1f/f91fcb9dd47a19b720fb48042a2f6f023651948e73726e98fff60d5ed5c7/torchaudio-2.10.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:da1081d1018a1e95f5a13947402aeb037cf5ac8861219a6164df004898a96bb1", size = 1897271, upload-time = "2026-01-21T16:28:23.519Z" }, - { url = "https://files.pythonhosted.org/packages/57/27/270c26890f43838e8faa5d3e52f079bd9d9d09f9a535a11cf6b94e20ed21/torchaudio-2.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f1afa53146a5655258d3a86e689c6879dfe78581d9bee9ef611ace98722f86bb", size = 478966, upload-time = "2026-01-21T16:28:32.491Z" }, +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/d9/357eb5fe4e19a861e6fa1af4d9f535e8fa8692336e6cf436e8a21262e054/torchaudio-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ebb59c694909eccb5d61b7cc199d297692012c43286e36d92983aa7bad7586d", size = 684145, upload-time = "2026-03-23T18:13:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/2a/79/90de77e73f395bba2fe477f8e82e4ae1d14d6452a706838765e850a5e80c/torchaudio-2.11.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:be7ad472acb16d16e98c005f0219b0db06a47dfe8f7b4d177062e1638f871e3b", size = 1626521, upload-time = "2026-03-23T18:13:40.98Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/5757ed7d8d11a6c14336bcb54e63980979f00005555fec80fb4aa4de5eff/torchaudio-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:5847fe2022b17c6580aeb39c8797a443411cc09edfd9183cd50ac1a3b8ccf97c", size = 1771929, upload-time = "2026-03-23T18:13:43.432Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f4/8ce2417eac66296e45b7aaa69858403fb6a52b1323f8635ec37b4b0f1fa3/torchaudio-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:7e2da1df4f6fe885c46db350a0dc90a0dff4b54541dff8846faa904d255e2bfe", size = 328661, upload-time = "2026-03-23T18:13:45.77Z" }, + { url = "https://files.pythonhosted.org/packages/94/77/0eec7f175d88f312296bd5b11c23bd58da37c1021f53da3db4df449ce3ee/torchaudio-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:492dd64645e9d0bb843e94f1d9a4d1e31426262ffc594fafecc1697df9df5eb9", size = 684142, upload-time = "2026-03-23T18:13:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f9/6f7ebe071b44592c85269762b55b63ab0a091b5f479f73544738f7564a1e/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:73dab4841f94d888bc7c2aed7b5547c643edc974306919fe1adfb65d57cccf4b", size = 1626527, upload-time = "2026-03-23T18:13:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/ac/70/17408e0d154d0c894537a88dcbadc48e8ad3b6e1ef4a1dabda5d40245ee0/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1a07ec72fd6f26a588c39b5f029e0130d16bb40bc4221635580bf8fb18fcbc80", size = 1771930, upload-time = "2026-03-23T18:13:37.963Z" }, + { url = "https://files.pythonhosted.org/packages/c9/75/b6d03fc75b409bdaec597274d1bdd4213db716ed16f6801386b31d59c551/torchaudio-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb59ba4452bbbe95d75ad3ef18df9824955625f36698ce9a5998a4a9f3c1ba1d", size = 328658, upload-time = "2026-03-23T18:13:44.545Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b1/77658817acacd01a72b714440c62f419efc4d90170e704e8e7a2c0918988/torchaudio-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1cf1acc883bee9cb906a933572fed6a8a933f86ef34e9ea7d803f72317e8c1b", size = 684226, upload-time = "2026-03-23T18:13:40.023Z" }, + { url = "https://files.pythonhosted.org/packages/78/28/c7adc053039f286c2aca0038b766cbe3294e66fec6b29a820e95128f9ede/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bc653defca1c16154398517a1adc98d0fb7f1dd08e58ced217558d213c2c6e29", size = 1626670, upload-time = "2026-03-23T18:13:42.162Z" }, + { url = "https://files.pythonhosted.org/packages/88/d8/d6d0f896e064aa67377484efef4911cdcc07bce2929474e1417cc0af18c2/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6503c0bdb29daf2e6281bb70ea2dfe2c3553b782b619eb5d73bdadd8a3f7cecf", size = 1771992, upload-time = "2026-03-23T18:13:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/23/a8/941277ecc39f7a0a169d554302a1f1afd87c1d94a8aec828891916cea59a/torchaudio-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:478110f981e5d40a8d82221732c57a56c85a1d5895fb8fe646e86ee15eded3bd", size = 328663, upload-time = "2026-03-23T18:13:19.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9e/f76fcd9877c8c78f258ee34e0fb8291fdb91e6218d582d9ca66b1e4bd4ae/torchaudio-2.11.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e3f9696a9ef1d49acc452159b052370c636406d072e9d8f10895fda87b591ea9", size = 679904, upload-time = "2026-03-23T18:13:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/249c1498ebdad3e7752866635ec0855fc0dcf898beccda5a9d2b9df8e4d0/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b034d7672f1c415434f48ef17807f2cce47f29e8795338c751d4e596c9fbe8b5", size = 1618523, upload-time = "2026-03-23T18:13:15.703Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/be13fe35d9aa5c26381c0e453c828a789d15c007f8f7d08c95341d19974d/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1c1101c1243ef0e4063ec63298977e2d3655c15cf88d9eb0a1bd4fe2db9f47ea", size = 1771992, upload-time = "2026-03-23T18:13:35.343Z" }, + { url = "https://files.pythonhosted.org/packages/e2/8b/2bbb3dca6ff28cba0de250874d5ef4fc2822c47a934b59b3974cff3219ef/torchaudio-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:986f4df5ed17b003dc52489468601720090e65f964f8bebccf90eb45bba75744", size = 328662, upload-time = "2026-03-23T18:13:18.308Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ce/52c652d30af7d6e96c8f1735d26131e94708e3f38d852b8fa97958804dd8/torchaudio-2.11.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:bda09ea630ae7207384fb0f28c35e4f8c0d82dd6eba020b6b335ad0caa9fed49", size = 680814, upload-time = "2026-03-23T18:13:17.08Z" }, + { url = "https://files.pythonhosted.org/packages/06/95/1ad1507482e7263e556709a3f5f87fecd375a0742cdaf238806c8e72eaad/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:9fe3083c62e035646483a14e180d33561bdc2eed436c9ab1259c137fb7120b4a", size = 1618546, upload-time = "2026-03-23T18:13:29.686Z" }, + { url = "https://files.pythonhosted.org/packages/98/4c/480328ba07487eb9890406720304d0d460dd7a6a64098614f5aa53b662ca/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:13cff988697ccbad539987599f9dc672f40c417bed67570b365e4e5002bbd096", size = 1771991, upload-time = "2026-03-23T18:13:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/3e/98/5d4790e2d6548768999acd34999d5aeefce8bcc23a07afaa5f03e723f557/torchaudio-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ed404c4399ad7f172c86a47c1b25293d322d1d58e26b10b0456a86cf67d37d84", size = 328661, upload-time = "2026-03-23T18:13:34.359Z" }, +] + +[[package]] +name = "torchcodec" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/7e/56dde94fd77a89c0ac289551ac91a39e99959bcd2ae9f14a3e928ca7ff9e/torchcodec-0.15.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:1cf1946806c419fa16e41c114ed783fe9f5115e08a02ed4c609c8e0402c6e70d", size = 4589746, upload-time = "2026-07-15T10:13:53.727Z" }, + { url = "https://files.pythonhosted.org/packages/55/af/f4cb8852016f8473d6ae89f5b7263b5c0fbf579405de87fa38a5b52b2ac5/torchcodec-0.15.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:db73e361d31bd7717d2861b5a41233ddefb29b3d1671cd61ca2b1eae5c0975c0", size = 2705124, upload-time = "2026-07-15T10:13:55.751Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/1cb65d491c5850706befffe0add082ad74f14ec1c3dda4f7bb45cd66796e/torchcodec-0.15.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:53cc91c1406f35055ce7077911f4a6fd43310ffe456d45d739afd37f547919ef", size = 2968579, upload-time = "2026-07-15T10:13:57.516Z" }, + { url = "https://files.pythonhosted.org/packages/eb/06/28b121a4a1a606e644606b8d01c4f2af70452342cd53fcebf412a936ed0f/torchcodec-0.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fd9665b76496d5453b7b7d0e832f142bc658b46fa2e7ae5a941f6dbb1752ea6", size = 3221667, upload-time = "2026-07-15T10:13:59.072Z" }, + { url = "https://files.pythonhosted.org/packages/cc/7e/1327cbf7fd2013ac2c06f438e6e6b4d4728b754d98e7c1539b80bafe5d42/torchcodec-0.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a1f089f44e237620a291078f3b039e3262c73bf0e5ff83747e20fd67e55d53d7", size = 4407134, upload-time = "2026-07-15T10:14:00.551Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2f/1ce2feb161bbd12719f7fd0accef64125f4a6d926e02cd360999b70429e4/torchcodec-0.15.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7ce1c120275f80eff56842b856ebd48232e46a87c6351607dd8d60ccf197bbb0", size = 2715872, upload-time = "2026-07-15T10:14:02.014Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ee/f14346b9e2d4aac6eee6f1658b5ffa1107e8c1be9e6a2130655a774755c6/torchcodec-0.15.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1c4257dc64554f1ed848511b7415b772f7df98217a30610d284e2a0a4df1a427", size = 2976495, upload-time = "2026-07-15T10:14:03.439Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b5/0ddc696f0121e5d300830cee43a73865a2436a8877833c97752cb005ed7b/torchcodec-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:80a918b840c346f4167781a8eab241aeaf7d22979a603960f99c1ed1fc992461", size = 3234179, upload-time = "2026-07-15T10:14:04.917Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/ba71ff29b3f957a7e05cfb5c1d189f34c4224166b5bbe900ec8320f506f7/torchcodec-0.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3a4b24012f7a7fe962dfee8f06d9c91e9e3fd1f4b6302fdb5b8884a02aca3f37", size = 4576065, upload-time = "2026-07-15T10:14:06.554Z" }, + { url = "https://files.pythonhosted.org/packages/9d/de/c00b8d13e3e28de9c76f05b4c25fc4d882b4a3d1451b8d2073d089895684/torchcodec-0.15.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5c62f4257b49c6473b0a1006519274b7daef9ef9c1d66b1a6a025dba9df5daac", size = 2727846, upload-time = "2026-07-15T10:14:08.066Z" }, + { url = "https://files.pythonhosted.org/packages/d0/05/b7ba7ae04db4afeb1fd32d30ec6290d511c374adc464afe191c8fc8d4e22/torchcodec-0.15.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fa31e33884829332cc55b301aa9d23ba90bf164aa8576a8c68aed6c0061c2d8c", size = 2988620, upload-time = "2026-07-15T10:14:09.504Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0f/fa432d8c8b523f5891a66483f607ec80e28ae025d99ce1d1c50667d8446b/torchcodec-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:589e127778870c691d8977c08311bf57c4fecb9eb56fa52cf29d9671fe78eb72", size = 3242793, upload-time = "2026-07-15T10:14:10.84Z" }, + { url = "https://files.pythonhosted.org/packages/4e/51/5edd02c5c5c511017a206c03818951b334a5d64e2fcf000a2fe17a026122/torchcodec-0.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7d753c456ff9c9c30f2b8862d08ce47074f96b1789fb60f12787a85f6c2c6a50", size = 4563016, upload-time = "2026-07-15T10:14:12.283Z" }, + { url = "https://files.pythonhosted.org/packages/af/b9/f9815625b201d41a934f8129d9cf8e956d0b7cccdbc39443538f4f582b38/torchcodec-0.15.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:28c8008494e47c3828eb64b2e9943dbb86d7183c3901a894eda26ce86e01b1a9", size = 2729991, upload-time = "2026-07-15T10:14:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/f0e5795100bdf11f6e73a2fcc5197e9010e45030c1ee7f6b3ee32cffefe4/torchcodec-0.15.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5896a55374d4c90e4788a8eb7c0c7a26a67db5bbe0b6c73975e2bd22f4d98fda", size = 2989745, upload-time = "2026-07-15T10:14:15.194Z" }, + { url = "https://files.pythonhosted.org/packages/75/46/60a7c34180bbe55410aa36783e4c7421d4fc81fccebb33dcbc6ac9d2a8a3/torchcodec-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:ed08f3f07f1c68c4123cf85a47a0947f5edd57da8cd1c10d46eefd4e1673a789", size = 3243199, upload-time = "2026-07-15T10:14:16.638Z" }, ] [[package]] name = "torchvision" -version = "0.25.0" +version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -4078,26 +4565,26 @@ dependencies = [ { name = "torch" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/50/ae/cbf727421eb73f1cf907fbe5788326a08f111b3f6b6ddca15426b53fec9a/torchvision-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a95c47abb817d4e90ea1a8e57bd0d728e3e6b533b3495ae77d84d883c4d11f56", size = 1874919, upload-time = "2026-01-21T16:27:47.617Z" }, - { url = "https://files.pythonhosted.org/packages/64/68/dc7a224f606d53ea09f9a85196a3921ec3a801b0b1d17e84c73392f0c029/torchvision-0.25.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:acc339aba4a858192998c2b91f635827e40d9c469d9cf1455bafdda6e4c28ea4", size = 2343220, upload-time = "2026-01-21T16:27:44.26Z" }, - { url = "https://files.pythonhosted.org/packages/f9/fa/8cce5ca7ffd4da95193232493703d20aa06303f37b119fd23a65df4f239a/torchvision-0.25.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0d9a3f925a081dd2ebb0b791249b687c2ef2c2717d027946654607494b9b64b6", size = 8068106, upload-time = "2026-01-21T16:27:37.805Z" }, - { url = "https://files.pythonhosted.org/packages/8b/b9/a53bcf8f78f2cd89215e9ded70041765d50ef13bf301f9884ec6041a9421/torchvision-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:b57430fbe9e9b697418a395041bb615124d9c007710a2712fda6e35fb310f264", size = 3697295, upload-time = "2026-01-21T16:27:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/3e/be/c704bceaf11c4f6b19d64337a34a877fcdfe3bd68160a8c9ae9bea4a35a3/torchvision-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db74a551946b75d19f9996c419a799ffdf6a223ecf17c656f90da011f1d75b20", size = 1874923, upload-time = "2026-01-21T16:27:46.574Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e9/f143cd71232430de1f547ceab840f68c55e127d72558b1061a71d0b193cd/torchvision-0.25.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f49964f96644dbac2506dffe1a0a7ec0f2bf8cf7a588c3319fed26e6329ffdf3", size = 2344808, upload-time = "2026-01-21T16:27:43.191Z" }, - { url = "https://files.pythonhosted.org/packages/43/ae/ad5d6165797de234c9658752acb4fce65b78a6a18d82efdf8367c940d8da/torchvision-0.25.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:153c0d2cbc34b7cf2da19d73450f24ba36d2b75ec9211b9962b5022fb9e4ecee", size = 8070752, upload-time = "2026-01-21T16:27:33.748Z" }, - { url = "https://files.pythonhosted.org/packages/23/19/55b28aecdc7f38df57b8eb55eb0b14a62b470ed8efeb22cdc74224df1d6a/torchvision-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:ea580ffd6094cc01914ad32f8c8118174f18974629af905cea08cb6d5d48c7b7", size = 4038722, upload-time = "2026-01-21T16:27:41.355Z" }, - { url = "https://files.pythonhosted.org/packages/56/3a/6ea0d73f49a9bef38a1b3a92e8dd455cea58470985d25635beab93841748/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2abe430c90b1d5e552680037d68da4eb80a5852ebb1c811b2b89d299b10573b", size = 1874920, upload-time = "2026-01-21T16:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/51/f8/c0e1ef27c66e15406fece94930e7d6feee4cb6374bbc02d945a630d6426e/torchvision-0.25.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b75deafa2dfea3e2c2a525559b04783515e3463f6e830cb71de0fb7ea36fe233", size = 2344556, upload-time = "2026-01-21T16:27:40.125Z" }, - { url = "https://files.pythonhosted.org/packages/68/2f/f24b039169db474e8688f649377de082a965fbf85daf4e46c44412f1d15a/torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f25aa9e380865b11ea6e9d99d84df86b9cc959f1a007cd966fc6f1ab2ed0e248", size = 8072351, upload-time = "2026-01-21T16:27:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/16/8f650c2e288977cf0f8f85184b90ee56ed170a4919347fc74ee99286ed6f/torchvision-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9c55ae8d673ab493325d1267cbd285bb94d56f99626c00ac4644de32a59ede3", size = 4303059, upload-time = "2026-01-21T16:27:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5b/1562a04a6a5a4cf8cf40016a0cdeda91ede75d6962cff7f809a85ae966a5/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:24e11199e4d84ba9c5ee7825ebdf1cd37ce8deec225117f10243cae984ced3ec", size = 1874918, upload-time = "2026-01-21T16:27:39.02Z" }, - { url = "https://files.pythonhosted.org/packages/36/b1/3d6c42f62c272ce34fcce609bb8939bdf873dab5f1b798fd4e880255f129/torchvision-0.25.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f271136d2d2c0b7a24c5671795c6e4fd8da4e0ea98aeb1041f62bc04c4370ef", size = 2309106, upload-time = "2026-01-21T16:27:30.624Z" }, - { url = "https://files.pythonhosted.org/packages/c7/60/59bb9c8b67cce356daeed4cb96a717caa4f69c9822f72e223a0eae7a9bd9/torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:855c0dc6d37f462482da7531c6788518baedca1e0847f3df42a911713acdfe52", size = 8071522, upload-time = "2026-01-21T16:27:29.392Z" }, - { url = "https://files.pythonhosted.org/packages/32/a5/9a9b1de0720f884ea50dbf9acb22cbe5312e51d7b8c4ac6ba9b51efd9bba/torchvision-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:cef0196be31be421f6f462d1e9da1101be7332d91984caa6f8022e6c78a5877f", size = 4321911, upload-time = "2026-01-21T16:27:35.195Z" }, - { url = "https://files.pythonhosted.org/packages/52/99/dca81ed21ebaeff2b67cc9f815a20fdaa418b69f5f9ea4c6ed71721470db/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a8f8061284395ce31bcd460f2169013382ccf411148ceb2ee38e718e9860f5a7", size = 1896209, upload-time = "2026-01-21T16:27:32.159Z" }, - { url = "https://files.pythonhosted.org/packages/28/cc/2103149761fdb4eaed58a53e8437b2d716d48f05174fab1d9fcf1e2a2244/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:146d02c9876858420adf41f3189fe90e3d6a409cbfa65454c09f25fb33bf7266", size = 2310735, upload-time = "2026-01-21T16:27:22.327Z" }, - { url = "https://files.pythonhosted.org/packages/76/ad/f4c985ad52ddd3b22711c588501be1b330adaeaf6850317f66751711b78c/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c4d395cb2c4a2712f6eb93a34476cdf7aae74bb6ea2ea1917f858e96344b00aa", size = 8089557, upload-time = "2026-01-21T16:27:27.666Z" }, - { url = "https://files.pythonhosted.org/packages/63/cc/0ea68b5802e5e3c31f44b307e74947bad5a38cc655231d845534ed50ddb8/torchvision-0.25.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5e6b449e9fa7d642142c0e27c41e5a43b508d57ed8e79b7c0a0c28652da8678c", size = 4344260, upload-time = "2026-01-21T16:27:17.018Z" }, + { url = "https://files.pythonhosted.org/packages/74/b4/cdfee31e0402ea035135462cb0ab496e974d56fab6b4e7a1f0cbccb8cd28/torchvision-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a06d4772a8e13e772906ed736cc53ec6639e5e60554f8e5fa6ca165aabebc464", size = 1863503, upload-time = "2026-03-23T18:13:01.384Z" }, + { url = "https://files.pythonhosted.org/packages/e4/74/11fee109841e80ad14e5ca2d80bff6b10eb11b7838ff06f35bfeaa9f7251/torchvision-0.26.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:2adfbe438473236191ff077a4a9a0c767436879c89628aa97137e959b0c11a94", size = 7766423, upload-time = "2026-03-23T18:12:56.049Z" }, + { url = "https://files.pythonhosted.org/packages/5e/00/24d8c7845c3f270153fb81395a5135b2778e2538e81d14c6aea5106c689c/torchvision-0.26.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b6f9ad1ecc0eab52647298b379ee9426845f8903703e6127973f8f3d049a798b", size = 7518249, upload-time = "2026-03-23T18:12:51.743Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/e53cd7c0da7ae002e5e929c1796ebbe7ec0c700c29f7a0a6696497fb3d8b/torchvision-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:f13f12b3791a266de2d599cb8162925261622a037d87fc03132848343cf68f75", size = 3669784, upload-time = "2026-03-23T18:12:49.949Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bd/d552a2521bade3295b2c6e7a4a0d1022261cab7ca7011f4e2a330dbb3caa/torchvision-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55bd6ad4ae77be01ba67a410b05b51f53b0d0ee45f146eb6a0dfb9007e70ab3c", size = 1863499, upload-time = "2026-03-23T18:12:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, + { url = "https://files.pythonhosted.org/packages/10/58/ed8f7754299f3e91d6414b6dc09f62b3fa7c6e5d63dfe48d69ab81498a37/torchvision-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:de6424b12887ad884f39a0ee446994ae3cd3b6a00a9cafe1bead85a031132af0", size = 3983834, upload-time = "2026-03-23T18:13:00.224Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a9/c272623a0f735c35f0f6cd6dc74784d4f970e800cf063bb76687895a2ab9/torchvision-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7993c01648e7c61d191b018e84d38fe0825c8fcb2720cd0f37caf7ba14404aa1", size = 4255155, upload-time = "2026-03-23T18:12:32.652Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/0762f77f53605d10c9477be39bb47722cc8e383bbbc2531471ce0e396c07/torchvision-0.26.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5d63dd43162691258b1b3529b9041bac7d54caa37eae0925f997108268cbf7c4", size = 1860809, upload-time = "2026-03-23T18:12:47.629Z" }, + { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, + { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/f1bc86a918c5f6feab1eeff11982e2060f4704332e96185463d27855bdf5/torchvision-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:4280c35ec8cba1fcc8294fb87e136924708726864c379e4c54494797d86bc474", size = 4319880, upload-time = "2026-03-23T18:12:38.168Z" }, + { url = "https://files.pythonhosted.org/packages/66/28/b4ad0a723ed95b003454caffcc41894b34bd8379df340848cae2c33871de/torchvision-0.26.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:358fc4726d0c08615b6d83b3149854f11efb2a564ed1acb6fce882e151412d23", size = 1951973, upload-time = "2026-03-23T18:12:48.781Z" }, + { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, + { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a4/f1155e943ae5b32400d7000adc81c79bb0392b16ceb33bcf13e02e48cced/torchvision-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ebc043cc5a4f0bf22e7680806dbba37ffb19e70f6953bbb44ed1a90aeb5c9bea", size = 4248202, upload-time = "2026-03-23T18:12:41.423Z" }, ] [[package]] @@ -4137,10 +4624,15 @@ name = "triton" version = "3.6.0" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/44/ba/b1b04f4b291a3205d95ebd24465de0e5bf010a2df27a4e58a9b5f039d8f2/triton-3.6.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781", size = 175972180, upload-time = "2026-01-20T16:15:53.664Z" }, { url = "https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea", size = 188105201, upload-time = "2026-01-20T16:00:29.272Z" }, + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, ] @@ -4278,25 +4770,26 @@ wheels = [ [[package]] name = "vllm" -version = "0.19.1" +version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, + { name = "apache-tvm-ffi" }, { name = "blake3" }, { name = "cachetools" }, { name = "cbor2" }, { name = "cloudpickle" }, { name = "compressed-tensors" }, { name = "depyf" }, - { name = "diskcache" }, { name = "einops" }, { name = "fastapi", extra = ["standard"] }, + { name = "fastsafetensors" }, { name = "filelock" }, - { name = "flashinfer-cubin" }, { name = "flashinfer-python" }, - { name = "gguf" }, + { name = "humming-kernels", extra = ["cu13"] }, { name = "ijson" }, + { name = "jsonschema" }, { name = "lark" }, { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, { name = "lm-format-enforcer" }, @@ -4308,7 +4801,8 @@ dependencies = [ { name = "numba" }, { name = "numpy" }, { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl" }, + { name = "nvidia-cutlass-dsl", extra = ["cu13"] }, + { name = "nvtx" }, { name = "openai" }, { name = "openai-harmony" }, { name = "opencv-python-headless" }, @@ -4326,20 +4820,26 @@ dependencies = [ { name = "py-cpuinfo" }, { name = "pybase64" }, { name = "pydantic" }, + { name = "pynvvideocodec" }, { name = "python-json-logger" }, { name = "pyyaml" }, { name = "pyzmq" }, { name = "quack-kernels" }, { name = "regex" }, { name = "requests" }, + { name = "safetensors" }, { name = "sentencepiece" }, { name = "setproctitle" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, { name = "six", marker = "python_full_version >= '3.12'" }, + { name = "starlette" }, { name = "tiktoken" }, + { name = "tilelang" }, { name = "tokenizers" }, + { name = "tokenspeed-mla", marker = "sys_platform == 'linux'" }, { name = "torch" }, { name = "torchaudio" }, + { name = "torchcodec" }, { name = "torchvision" }, { name = "tqdm" }, { name = "transformers" }, @@ -4347,10 +4847,10 @@ dependencies = [ { name = "watchfiles" }, { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a8/49/60a2a962ecbf780c8fbfd0d5548b208d654d5c4267df94d8d93883641431/vllm-0.19.1.tar.gz", hash = "sha256:9fb88ce6b50991eba41d183584f65f51d7f6015d86a42cdabf79c1c8bd5d66fa", size = 31105401, upload-time = "2026-04-18T05:50:15.143Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/72/fa30f8459d11ae206f1a20bd0ac7ed1b9e390b695fa3dfc9ef6056de0cfe/vllm-0.26.0.tar.gz", hash = "sha256:23e9fa19d7e20ce7dcc1c074d41503e2116d23f19e688f5d5ea91b741f958502", size = 38353572, upload-time = "2026-07-25T10:40:48.095Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/28/4c/26c426103c58ac8d98435fe63c7758a2f289b5481a08be19e9c9fe29a4c2/vllm-0.19.1-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:c8dde3c9af20f00a644e64a50ebe43948f2921bab3ffd5407d634c15836cb181", size = 385252556, upload-time = "2026-04-18T05:49:16.101Z" }, - { url = "https://files.pythonhosted.org/packages/78/20/f41216b79c87372a9d03175f36fa1411ee61059ce8c557d2691722ea4aae/vllm-0.19.1-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:71a87f46cafab4489c69a5c5c83b870d0235e5694d8222303d460576293dc719", size = 433132101, upload-time = "2026-04-18T05:49:54.202Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/6ff13689a5931f0c97b7008042f07aacc4a246e7eb06fd9b4d5a72de483c/vllm-0.26.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:52a4c3e55c2c80cc8793e52ccc244457ceade25b0ad7caa1c15e5002a95a1b2c", size = 298269785, upload-time = "2026-07-25T10:40:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/20/96/86edd288415aafc2952bbb969b5ef4e8c58e5525185b60320730276921e6/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adb1e4c9b46d0dfdb094121ae5aad670a42412dd813ed4e5db069ed6a15006de", size = 303698761, upload-time = "2026-07-25T10:40:32.107Z" }, ] [[package]] @@ -4371,7 +4871,7 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "vllm", marker = "extra == 'vllm'", specifier = "==0.19.1" }] +requires-dist = [{ name = "vllm", marker = "extra == 'vllm'", specifier = "==0.26.0" }] provides-extras = ["vllm"] [package.metadata.requires-dev] @@ -4523,7 +5023,7 @@ wheels = [ [[package]] name = "xgrammar" -version = "0.2.0" +version = "0.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -4534,27 +5034,27 @@ dependencies = [ { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a0/54/7e593fc41ffcaf5ac7c0379e0aec0cf03e53a742d1a91f64c6c7e79a6ac1/xgrammar-0.2.0.tar.gz", hash = "sha256:c4f0238a89869343171d43d069b8c5da874f3c2c25f408f20cd5987219a6adef", size = 2421093, upload-time = "2026-05-01T18:33:54.474Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/43/e19db659e9e56d88f3769f4052a955213cf6ad2341b7f1d583da83e361eb/xgrammar-0.2.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fc4fb16e99f807d5c8ca1926a43024a09d82331ae56e0fe14a85f365fc12f2f1", size = 23150211, upload-time = "2026-05-01T18:31:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/9c/eb/a265fcd2f18e5c2cf343079b857d081889d1dc080c524b9fdffda4040f16/xgrammar-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa89d9eb2d989b90b19ff0d0fa9cf9be05e970e56ad2e979315772496f7fe8bb", size = 23055199, upload-time = "2026-05-01T18:32:00.337Z" }, - { url = "https://files.pythonhosted.org/packages/2a/56/f5da0311502ed14b4a25988210bf366d470a2e3c91ecb8fb4499400ef7f6/xgrammar-0.2.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6130eacd86161c2eae8bd46a7108c0856757bbc8e5efd207eba36fc82db541f", size = 44155197, upload-time = "2026-05-01T18:32:03.69Z" }, - { url = "https://files.pythonhosted.org/packages/dd/b0/d776e41054932b9fcee8204fa44c9ffe42469b6414410270ccc09662142d/xgrammar-0.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e5daf9ab32850fce44eed894a2704ca91d537bf1e41943f8d11145c235eaa87", size = 44616365, upload-time = "2026-05-01T18:32:08.382Z" }, - { url = "https://files.pythonhosted.org/packages/f5/17/90074d932da98424476ce5e65f1243e9d5ddd95b03060d02fd125b2df18a/xgrammar-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:8a0aaa6cb71efeeb6223f06fa0b16829d8e7f9e5af51803899c2cca8ea8c7ba7", size = 7400272, upload-time = "2026-05-01T18:32:12.02Z" }, - { url = "https://files.pythonhosted.org/packages/66/76/82fa277ac2336bb21f2b3b6117e73081f184e6244544e61b57ba5b5f13c3/xgrammar-0.2.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c606b0e6deb328f9e7cfd2ae15232efca58a88c8dbe56447daccc63f89078c4f", size = 23150194, upload-time = "2026-05-01T18:32:14.314Z" }, - { url = "https://files.pythonhosted.org/packages/2e/f2/166a5afbf6a236a5044a5f3f56a271781417cbd4c406aad7427c5f6da8a0/xgrammar-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:47815edecad20eed1b1084ca97ebc2f8c2ba054bd0ee3cb2d5dd5681e781b632", size = 23055210, upload-time = "2026-05-01T18:32:17.183Z" }, - { url = "https://files.pythonhosted.org/packages/a2/f8/2122b33a44be20ee1466360c6916816b9a79ac38f430cd56676484614443/xgrammar-0.2.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:001e2177bd80bb7c49dca3a70a8c2a645c664afc03c3cad7abffc9340c9a4eff", size = 44155235, upload-time = "2026-05-01T18:32:21.288Z" }, - { url = "https://files.pythonhosted.org/packages/f0/bd/4c1598e93e1e9a6dcc650e57600a80b52d6d759f8f53b902ea34727bd6fe/xgrammar-0.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f03bcbd6cfd96864d59d8acd18e9e5a3f1656beedcdc55a553bf078120758ac", size = 44616355, upload-time = "2026-05-01T18:32:25.174Z" }, - { url = "https://files.pythonhosted.org/packages/b2/ca/6607c78f1c9aca915a109d6dae582da0f24da2de7f6d0ee426d2c6ee17af/xgrammar-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:8ea6c01a536e2317f8e2e21762567f504d54c198bdf936cf89abc7cc1448268b", size = 7400485, upload-time = "2026-05-01T18:32:28.796Z" }, - { url = "https://files.pythonhosted.org/packages/23/3c/c76e711834600226831666d6a54b3a139bf0512c90268b44b6fda69aaee2/xgrammar-0.2.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:fd525fbff2cd0541720c304cf9b2299c52e9d0645eb623816b411b8dcea64d1f", size = 23150208, upload-time = "2026-05-01T18:32:31.413Z" }, - { url = "https://files.pythonhosted.org/packages/02/78/5a5faa8ac7d6b6dd4dcfcc31bb1ab4c5a2f10ee90224fb3891181c76e24e/xgrammar-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f70f6c2f062535f54836851794d11f96f2a15d8e42a4b21dfbc14767d52c096e", size = 23055200, upload-time = "2026-05-01T18:32:34.927Z" }, - { url = "https://files.pythonhosted.org/packages/b7/1c/92eac0cd125ba195e3f1e3e25e89aedcaecbf99a4034ab12b7655ac07453/xgrammar-0.2.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddad831bc7da41d52ed34b7e1050c9a37d3f5f2314eaed8e658cbd2a34625e31", size = 44155238, upload-time = "2026-05-01T18:32:38.679Z" }, - { url = "https://files.pythonhosted.org/packages/7e/30/99f4e83821db16d58dd41249ba46038ed47bce274c57ad5567030775fc62/xgrammar-0.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a36c744d24d93e178c138486aa02b390a80326b64ff11e222e063a028dd65849", size = 44616361, upload-time = "2026-05-01T18:32:42.536Z" }, - { url = "https://files.pythonhosted.org/packages/1e/5c/35a84b53a057b637d60ee039872589204724c92579c1ded1bd7f8f1b449e/xgrammar-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:9bf78d76afcb94372b5c3a05da7f80ad74a8973d971d43a0d4961ea672a8f5fb", size = 7400441, upload-time = "2026-05-01T18:32:45.831Z" }, - { url = "https://files.pythonhosted.org/packages/f7/a1/ce7a1c2ebe89ee2715885935721169ecb9fbd13ab791d8f4dc0a86b87259/xgrammar-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4c7d88530100b6bbb0bc3ccfa7fb50385c938ce15e9219b47bb91fd5b63c2788", size = 23055213, upload-time = "2026-05-01T18:32:48.649Z" }, - { url = "https://files.pythonhosted.org/packages/36/22/18bfae3275613493f0fcbd274f2fa169f85c333ffa9581fca83c25669b8a/xgrammar-0.2.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ea1451a1df7aeb39ef97f7b4b8860b7f80424251943563aac48fa98b7b7e939", size = 44155210, upload-time = "2026-05-01T18:32:52.201Z" }, - { url = "https://files.pythonhosted.org/packages/5d/b5/0e4d77b7a91be685e7e388d06c7215cbb7c241402f64b4366d8a4a7a847e/xgrammar-0.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91b3cd498713042ae51c458e2357954e54df0abaea217d6e4297e8065f31a258", size = 44616344, upload-time = "2026-05-01T18:32:56.214Z" }, - { url = "https://files.pythonhosted.org/packages/0e/f6/974cb6cf9f2b62ec32d525af160cdea1b99f902c54b5e44c2c659a96ffc9/xgrammar-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:c85eef94b3905216fecd2c07d9b0c49d963280dfb7905636b44458b8add42ce0", size = 7400474, upload-time = "2026-05-01T18:32:59.597Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/81/f4/e71693d8cec60b7e36dab660784ecc5a6aa51e478a83b556011645c58c87/xgrammar-0.2.3.tar.gz", hash = "sha256:f76423630ae3ac4e090cb38ce1e30e7bcc69b3dee4d22d94353944386a4c6f18", size = 2447704, upload-time = "2026-06-27T04:45:24.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/e2/54bf9a915665d380ca09ba86b2449ceaed2500e11bd3654e0df3f9621e19/xgrammar-0.2.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:5eec3987abb915b7182587cf8063ae16b9fcb51e8b5f70558d8d2f0bfb8e04c4", size = 23284501, upload-time = "2026-06-27T04:43:56.188Z" }, + { url = "https://files.pythonhosted.org/packages/a1/80/ee5e79e79a1ef19aec4ad26155ce7f3cb99b474d62dad2d5a795035896cb/xgrammar-0.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f1270736d74ca3276cfba593457c23b9ca37032bd8051614533c81ed892d2727", size = 23240063, upload-time = "2026-06-27T04:43:59.283Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b4/d828e4d8b98b256609c18fd80b7b91c6b3c0b401b82830fb80b4ac36664b/xgrammar-0.2.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e983c6521ebac727e8513acd3f3eb76b9bfe9fedb3c528ddb01845c08f098652", size = 44314489, upload-time = "2026-06-27T04:44:02.863Z" }, + { url = "https://files.pythonhosted.org/packages/90/4d/2e8cf58db0ec4c31bf1a2370ec53987b1c5274b51003b777e2d0cd0e25b2/xgrammar-0.2.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b257973b2878bcb2c02f24057d174c0d9715cdc02e8ce29149a350184e619f86", size = 44855095, upload-time = "2026-06-27T04:44:06.242Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6b/e08774188ffaa04773048de4a326a228241a28d702a3db8ea104fe8f292c/xgrammar-0.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:ea79d53314d614a7beab09570a659ec72e4c79fe61b0a9ad153da95b4ee8fcd7", size = 15780258, upload-time = "2026-06-27T04:44:08.607Z" }, + { url = "https://files.pythonhosted.org/packages/6f/78/8664d2c92ffae29af09d98305432c5db276490f9ced87a4c4f054a60b606/xgrammar-0.2.3-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:00f6ba916fe84552f303b1b576061296ff8bea1e24d065efec49543489dd5217", size = 23284497, upload-time = "2026-06-27T04:44:10.736Z" }, + { url = "https://files.pythonhosted.org/packages/ae/24/e87485e32f9c1942dd56c66ce9e264784970eafc8e3c41ddd8f8daf916e0/xgrammar-0.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bec963362548e48b9a763de8f801d2b6c3f6dfb050cc88de2a83fe5e90dec357", size = 23240042, upload-time = "2026-06-27T04:44:13.618Z" }, + { url = "https://files.pythonhosted.org/packages/19/b5/f1b54a6f652cbd9dead9a6e31c8eedeea846abae020fb7eb08cc90b13786/xgrammar-0.2.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48d2c9d2bab9b60653204bf334663e06f6044d8f5c104aca68ee23526afb3161", size = 44314487, upload-time = "2026-06-27T04:44:16.182Z" }, + { url = "https://files.pythonhosted.org/packages/84/f3/4bd6bd3dd9a0450d78bcd9db3593b5df9ab5baf62a8f50d265a933156c47/xgrammar-0.2.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d38fb3ad3118b8f08b1da53fb6feb81a206e6eb9df6abb16bda47e2cb272deef", size = 44855068, upload-time = "2026-06-27T04:44:18.898Z" }, + { url = "https://files.pythonhosted.org/packages/9e/56/e7b1217befca273568efd6da80b2f1f7471633a5e7d6e2a9a1e87e13e490/xgrammar-0.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:8c262af85340825e6bda407293713582f95169fb098cb107751b55c0b3c309fb", size = 15780256, upload-time = "2026-06-27T04:44:21.562Z" }, + { url = "https://files.pythonhosted.org/packages/51/1c/0cdb22fc799e6d158b3243eeb895ae2e086825487b57767838c98d4864ee/xgrammar-0.2.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:173e167d43a5cf4171eee2be86097decff8803b0a0853d7baaf446c732a7d3a9", size = 23284489, upload-time = "2026-06-27T04:44:23.927Z" }, + { url = "https://files.pythonhosted.org/packages/cf/bc/994dc6f222189174840c29a1f5b4c175e69dfe13ed2e25b6dbbe9f200a29/xgrammar-0.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aa35f24835a59c822e249ecc80912eea4de03fc8b04afb2f82c8b950a56be6ef", size = 23240027, upload-time = "2026-06-27T04:44:26.255Z" }, + { url = "https://files.pythonhosted.org/packages/e4/79/0bb37937bf847c738c64b64dc50ddc12e7c526b34c5ab82cebe58da5ec8f/xgrammar-0.2.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11255f184971489fc72b948b096e2917f482ba2dca975177f5411562cedb9c6d", size = 44314481, upload-time = "2026-06-27T04:44:28.875Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fd/5ebd5d14b8993cb225151bbb8f2011742fc7a7d94a3bdbc3ec3954b9b62d/xgrammar-0.2.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdf081fab29694302d41d61dcf52fad7d253879a718bc6afc68db0a0dabd7f19", size = 44855110, upload-time = "2026-06-27T04:44:31.586Z" }, + { url = "https://files.pythonhosted.org/packages/17/66/67239f43b0244f65aec4639f51ab95905db42eb66e532ee2a4e5cdce32de/xgrammar-0.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:e7787dd8321a04f86116b756aa3dadd622e3607a3559b1e986cc5f77da00d68e", size = 15780277, upload-time = "2026-06-27T04:44:34.081Z" }, + { url = "https://files.pythonhosted.org/packages/1b/06/a1a3c3a53cebb5f1bdaa1d1452f3327b6ca7413bdeff94ff639c3b9b8378/xgrammar-0.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11108010c54c8f12f0b14c239ef5bca217cf107fa3d0bf0db008bf594ab0a1ff", size = 23240039, upload-time = "2026-06-27T04:44:36.173Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/1c74ff8bad6624c08a33924f7d051a9278a622b48d843e939d574a6b5bf8/xgrammar-0.2.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eeb5e46bd7d3230e5d8e6385793c48ec872a4fb377c19d341dbcaedc41f495e9", size = 44314496, upload-time = "2026-06-27T04:44:38.552Z" }, + { url = "https://files.pythonhosted.org/packages/28/f8/2407b44049416650c257e22399c32788d3497524a9a899addc74b5d27d1d/xgrammar-0.2.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d0fbd4709733b224e6e115d1001fb07124c71fe114a9087a2e3e53ce871517", size = 44855037, upload-time = "2026-06-27T04:44:41.122Z" }, + { url = "https://files.pythonhosted.org/packages/92/67/271dd31308c5a1d32d9c696044f9d44589f8ac4406faaddc3047765e0c71/xgrammar-0.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:3a718fa2c1bfc06951e1c480a656c28bde6d5f828199308fd577bd264e05b923", size = 15780261, upload-time = "2026-06-27T04:44:43.573Z" }, ] [[package]] @@ -4661,6 +5161,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, ] +[[package]] +name = "z3-solver" +version = "4.15.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/8e/0c8f17309549d2e5cde9a3ccefa6365437f1e7bafe71878eaf9478e47b18/z3_solver-4.15.4.0.tar.gz", hash = "sha256:928c29b58c4eb62106da51c1914f6a4a55d0441f8f48a81b9da07950434a8946", size = 5018600, upload-time = "2025-10-29T18:12:03.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/33/a3d5d2eaeb0f7b3174d57d405437eabb2075d4d50bd9ea0957696c435c7b/z3_solver-4.15.4.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:407e825cc9211f95ef46bdc8d151bf630e7ab2d62a21d24cd74c09cc5b73f3aa", size = 37052538, upload-time = "2025-10-29T18:11:46.233Z" }, + { url = "https://files.pythonhosted.org/packages/47/84/fd7ffac1551cd9f8d44fe41358f738be670fc4c24dfd514fab503f2cf3e7/z3_solver-4.15.4.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:00bd10c5a6a5f6112d3a9a810d0799227e52f76caa860dafa5e00966bb47eb13", size = 39807925, upload-time = "2025-10-29T18:11:49.81Z" }, + { url = "https://files.pythonhosted.org/packages/21/c9/bb51a96af0091324c81b803f16c49f719f9f6ea0b0bb52200f5c97ec4892/z3_solver-4.15.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e103a6f203f505b8b8b8e5c931cc407c95b61556512d4921c1ddc0b3f41b08e", size = 29268352, upload-time = "2025-10-29T18:11:53.032Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2e/0b49f7e4e53817cfb09a0f6585012b782dfe0b666e8abefcb4fac0570606/z3_solver-4.15.4.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:62c7e9cbdd711932301f29919ad9158de9b2f58b4d281dd259bbcd0a2f408ba1", size = 27226534, upload-time = "2025-10-29T18:11:55.59Z" }, + { url = "https://files.pythonhosted.org/packages/26/91/33de49538444d4aafbe47415c450c2f9abab1733e1226f276b496672f46c/z3_solver-4.15.4.0-py3-none-win32.whl", hash = "sha256:be3bc916545c96ffbf89e00d07104ff14f78336e55db069177a1bfbcc01b269d", size = 13191672, upload-time = "2025-10-29T18:11:58.424Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/a0b135e4419df475177ae78fc93c422430b0fd8875649486f9a5989772e6/z3_solver-4.15.4.0-py3-none-win_amd64.whl", hash = "sha256:00e35b02632ed085ea8199fb230f6015e6fc40554a6680c097bd5f060e827431", size = 16259597, upload-time = "2025-10-29T18:12:01.14Z" }, +] + [[package]] name = "zipp" version = "3.23.1" From e6e49417d53b6cdcd00198485e4e35752b073bef Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Sun, 2 Aug 2026 10:50:15 +0800 Subject: [PATCH 02/10] feat(npu): support vLLM 0.26 Signed-off-by: jiangkuaixue123 --- .../compat/patches/npu/force_load_balance.py | 352 +++++++----------- .../model_executor/models/deepseek_v2.py | 7 + .../v1/worker/npu/attention_model_runner.py | 11 - .../compat/patches/test_force_load_balance.py | 323 ++++++++-------- .../models/test_deepseek_v2_construction.py | 22 ++ 5 files changed, 344 insertions(+), 371 deletions(-) diff --git a/afd_plugin/compat/patches/npu/force_load_balance.py b/afd_plugin/compat/patches/npu/force_load_balance.py index 5ef64008..4e0a1259 100644 --- a/afd_plugin/compat/patches/npu/force_load_balance.py +++ b/afd_plugin/compat/patches/npu/force_load_balance.py @@ -19,21 +19,25 @@ from typing import Any import torch -import vllm_ascend.envs as envs_ascend import vllm_ascend.ops.fused_moe.fused_moe as fused_moe_module -from vllm.config import VllmConfig -from vllm_ascend.ascend_forward_context import _EXTRA_CTX, MoECommType -from vllm_ascend.flash_common3_context import get_flash_common3_context +from vllm.config import CompilationMode, VllmConfig, get_current_vllm_config +from vllm.logger import logger +from vllm_ascend.ascend_config import get_ascend_config +from vllm_ascend.ascend_forward_context import ( + _EXTRA_CTX, + _MEGA_MOE_SUPPORTED, + MoECommType, +) +from vllm_ascend.distributed.parallel_state import get_mc2_group from vllm_ascend.ops.fused_moe.experts_selector import ( select_experts, zero_experts_compute, ) -from vllm_ascend.ops.fused_moe.fused_moe import AscendFusedMoE +from vllm_ascend.ops.fused_moe.moe_runtime_args import build_fused_experts_input +from vllm_ascend.quantization.methods.base import get_moe_num_logical_experts from vllm_ascend.quantization.methods.w8a8_dynamic import ( AscendW8A8DynamicFusedMoEMethod, - build_fused_experts_input, ) -from vllm_ascend.quantization.quant_type import QuantType _FORCE_LB_DETERMINISTIC_SEED = 1024 @@ -65,16 +69,6 @@ def _get_force_lb_max_tokens(vllm_config: VllmConfig) -> int: return max(max_tokens, 1) -def _get_force_lb_config(layer: object) -> ForceLoadBalanceConfig: - return ForceLoadBalanceConfig( - n_routed_experts=int(layer.n_routed_experts), - ep_size=int(layer.ep_size), - ep_rank=int(layer.ep_rank), - top_k=int(layer.top_k), - topn_per_rank=int(layer.force_load_balance_topn_per_rank), - ) - - def _validate_force_lb_config(config: ForceLoadBalanceConfig) -> None: assert config.ep_size > 0, "ep_size must be positive" assert 0 <= config.ep_rank < config.ep_size, ( @@ -143,16 +137,16 @@ def _build_topk_buffer( def _init_force_lb_buffer( - layer: object, + method: AscendW8A8DynamicFusedMoEMethod, + config: ForceLoadBalanceConfig, max_tokens: int, device: torch.device, ) -> None: - config = _get_force_lb_config(layer) _validate_force_lb_config(config) buffer = _build_topk_buffer(config, max_tokens, device) - layer.force_lb_fake_topk_buffer = buffer - layer.max_force_lb_tokens = max_tokens + method.force_lb_fake_topk_buffer = buffer + method.max_force_lb_tokens = max_tokens fused_moe_module.logger.info( "AFD force load balance buffer initialized: ep_size=%s top_k=%s" @@ -166,11 +160,12 @@ def _init_force_lb_buffer( def _get_force_lb_topk_ids( - layer: object, + method: AscendW8A8DynamicFusedMoEMethod, + config: ForceLoadBalanceConfig, batch_tokens: int, device: torch.device, ) -> torch.Tensor: - buffer: torch.Tensor | None = getattr(layer, "force_lb_fake_topk_buffer", None) + buffer = method.force_lb_fake_topk_buffer if buffer is None: raise RuntimeError("force_lb_fake_topk_buffer is not initialized") @@ -181,71 +176,48 @@ def _get_force_lb_topk_ids( buffer.size(0), new_max_tokens, ) - _init_force_lb_buffer(layer, new_max_tokens, device) - buffer = layer.force_lb_fake_topk_buffer + _init_force_lb_buffer(method, config, new_max_tokens, device) + buffer = method.force_lb_fake_topk_buffer + assert buffer is not None if buffer.device != device: buffer = buffer.to(device, non_blocking=True) - layer.force_lb_fake_topk_buffer = buffer + method.force_lb_fake_topk_buffer = buffer - top_k = int(layer.top_k) - return buffer[:batch_tokens, :top_k] + return buffer[:batch_tokens, : config.top_k] -# Patch reason: vllm-ascend's AscendFusedMoE does not initialize AFD profiling -# knobs for deterministic force-load-balance routing. -# Patch functionality: preserves the target upstream tag's AscendFusedMoE -# initialization and replaces the force-load-balance buffer setup with AFD's -# deterministic fake top-k buffer. +# Patch reason: vllm-ascend's W8A8 method does not retain AFD's deterministic +# force-load-balance settings after leaving the model-construction config context. +# Patch functionality: preserves upstream initialization and captures the AFD +# profiling switch, local-expert limit, and initial buffer capacity for apply. # Signature: matches upstream; no added parameters. -def __init__(self, *args, **kwargs): - super(AscendFusedMoE, self).__init__(*args, **kwargs) - - num_experts = kwargs["num_experts"] - intermediate_size = kwargs["intermediate_size"] - num_shared_experts = kwargs.get("n_shared_experts", 0) - self.n_routed_experts = num_experts - - AscendFusedMoE.moe_counter += 1 - self.moe_instance_id = AscendFusedMoE.moe_counter - - self._expert_map = None - self.log2phy = None - - if self.quant_config is None: - self.quant_method = fused_moe_module.AscendUnquantizedFusedMoEMethod( - self.moe_config - ) - else: - self.quant_method = self.quant_config.get_quant_method(self, self.layer_name) - - assert self.quant_method is not None - - self.moe_config.tp_group = fused_moe_module.get_tp_group() - self.moe_config.dp_group = fused_moe_module.get_dp_group() - self.moe_config.ep_group = fused_moe_module.get_ep_group() - self.moe_config.mc2_group = fused_moe_module.get_mc2_group() - self.moe_config.supports_eplb = self.quant_method.supports_eplb - ascend_config = fused_moe_module.get_ascend_config() - vllm_config = fused_moe_module.get_current_vllm_config() - additional_config = getattr(vllm_config, "additional_config", None) - if not isinstance(additional_config, dict): - additional_config = {} - # flashcommon3 gate stream - self.multistream_overlap_gate = ascend_config.multistream_overlap_gate - if self.multistream_overlap_gate and AscendFusedMoE.gate_stream is None: - AscendFusedMoE.gate_stream = torch.npu.Stream() - if ( - self.custom_routing_function is None - and self.e_score_correction_bias is not None - ): - self.e_score_correction_bias.data = self.e_score_correction_bias.data.to( - dtype=vllm_config.model_config.dtype +def __init__(self): + vllm_config = get_current_vllm_config() + ascend_config = get_ascend_config() + self.use_aclgraph = ( + vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE + and not vllm_config.model_config.enforce_eager + ) + self.dynamic_eplb = ascend_config.eplb_config.dynamic_eplb + self.in_dtype = vllm_config.model_config.dtype + self.supports_eplb = True + + try: + device_group = get_mc2_group().device_group + # TODO: Try local_rank = ep_group.rank_in_group + local_rank = torch.distributed.get_rank(group=device_group) + backend = device_group._get_backend(torch.device("npu")) + self.moe_all_to_all_group_name = backend.get_hccl_comm_name(local_rank) + except AttributeError: + logger.warning_once( + "[vllm-ascend/W8A8_DYNAMIC] MC2 group metadata unavailable, " + "falling back to empty moe_all_to_all_group_name." ) + self.moe_all_to_all_group_name = "" - # ### PATCH START: AFD force-load-balance layer initialization - # Read plugin-owned profiling knobs and prebuild deterministic fake routed - # expert ids for Ascend W8A8 MoE layers. + # ### PATCH START: capture AFD force-load-balance configuration + additional_config = vllm_config.additional_config or {} self.enable_force_load_balance = bool( additional_config.get("enable_force_load_balance", False) ) @@ -253,109 +225,14 @@ def __init__(self, *args, **kwargs): additional_config.get("force_load_balance_topn_per_rank", 0) ) self.max_force_lb_tokens = _get_force_lb_max_tokens(vllm_config) - self.force_lb_fake_topk_buffer = None - # ### PATCH END: AFD force-load-balance layer initialization - - # init moe - eplb_config = ascend_config.eplb_config - self.mix_placement = getattr(ascend_config, "mix_placement", False) - self.n_shared_experts = num_shared_experts - num_experts += num_shared_experts if self.mix_placement else 0 - self.moe_config.num_experts = num_experts - ( - self.global_expert_map, - self._expert_map, - self.log2phy, - self.global_redundant_expert_num, - ) = fused_moe_module.init_eplb_config( - eplb_config, - self.moe_instance_id, - self.moe_config, - self.mix_placement, - num_shared_experts, - ) - self.global_num_experts = num_experts + self.global_redundant_expert_num - self.dynamic_eplb = eplb_config.dynamic_eplb and (self.log2phy is not None) - self.local_num_experts = self.global_num_experts // self.ep_size - if self._expert_map is not None: - fused_moe_module.logger.info_once( - "[EP Rank %s/%s] Expert parallelism is enabled. Local/global" - " number of experts: %s/%s. Experts local to global index map:" - " %s.", - self.ep_rank, - self.ep_size, - self.local_num_experts, - self.global_num_experts, - fused_moe_module.get_compressed_expert_map(self._expert_map), - ) - if self.dynamic_eplb: - self.multi_stage = False - self.moe_load = torch.zeros(self.local_num_experts, dtype=torch.int64).npu() - if eplb_config.eplb_policy_type == 3: - self.multi_stage = True - self.load_counter = torch.tensor(0, dtype=torch.int32, device="npu") - self.num_iter = eplb_config.expert_heat_collection_interval - self.moe_load = torch.zeros( - (self.num_iter, self.local_num_experts), - dtype=torch.int32, - device="npu", - ) - - self.moe_config.num_experts = self.global_num_experts - self.moe_config.num_local_experts = self.local_num_experts - self.moe_config.global_redundant_expert_num = self.global_redundant_expert_num - - moe_quant_params = { - "num_experts": self.local_num_experts, - "hidden_size": self.hidden_size, - "intermediate_size_per_partition": self.intermediate_size_per_partition, - "params_dtype": self.params_dtype, - "weight_loader": self.weight_loader, - } - # need full intermediate size pre-sharding for WNA16 act order - if self.quant_method.__class__.__name__ in ( - "GPTQMarlinMoEMethod", - "CompressedTensorsWNA16MoEMethod", - ): - moe_quant_params["intermediate_size_full"] = intermediate_size - self.quant_method.create_weights(layer=self, **moe_quant_params) - - self.enable_shared_expert_dp = ascend_config.enable_shared_expert_dp - self.enable_npugraph_ex_static_kernel = ( - ascend_config.ascend_compilation_config.enable_static_kernel - ) - - fused_moe_module.setup_moe_comm_method(self.moe_config) - self.quant_type = self._get_quant_type() - - # ### PATCH START: AFD force-load-balance layer initialization - # Initialize the deterministic fake top-k buffer after W8A8 weights exist. - if self.enable_force_load_balance and self.quant_type == QuantType.W8A8: - _init_force_lb_buffer( - self, - int(self.max_force_lb_tokens), - self.w13_weight.device, - ) - # ### PATCH END: AFD force-load-balance layer initialization - - is_legacy = fused_moe_module.vllm_version_is("0.19.1") - self.runner = fused_moe_module.AscendMoERunner( - self if is_legacy else self.layer_name, - self.moe_config, - self.router, - self._routed_input_transform, - self.gate if is_legacy else kwargs.pop("gate", None), - self.shared_experts if is_legacy else kwargs.pop("shared_experts", None), - self.quant_method, - self.reduce_results, - self.vllm_config.parallel_config.enable_dbo, - ) + self.force_lb_fake_topk_buffer: torch.Tensor | None = None + # ### PATCH END: capture AFD force-load-balance configuration # Patch reason: vllm-ascend W8A8 MoE routes tokens with model-selected expert # ids, but AFD profiling needs deterministic balanced expert ids. # Patch functionality: preserves the target upstream tag's W8A8 apply path and -# replaces only layer-owned force-load-balance top-k ids with AFD deterministic +# replaces model-selected top-k ids with the method-owned AFD deterministic # ids. # Signature: matches upstream; no added parameters. def apply( @@ -382,6 +259,7 @@ def apply( activation: str = "silu", apply_router_weight_on_input: bool = False, mc2_mask: torch.Tensor | None = None, + tid2eid: torch.Tensor | None = None, ) -> torch.Tensor: zero_expert_num = getattr(layer, "zero_expert_num", 0) zero_expert_type = getattr(layer, "zero_expert_type", None) @@ -389,42 +267,47 @@ def apply( mix_placement = getattr(layer, "mix_placement", False) if n_shared_experts is None: n_shared_experts = 0 - valid_global_expert_num = num_experts - n_shared_experts + num_logical_experts = get_moe_num_logical_experts( + layer, + num_experts, + global_redundant_expert_num=global_redundant_expert_num, + num_shared_experts=n_shared_experts, + ) if zero_expert_num == 0 or zero_expert_type is None: - assert router_logits.shape[1] == valid_global_expert_num, ( - "Number of global experts mismatch (excluding redundancy)" + assert router_logits.shape[1] == num_logical_experts, ( + "[vllm-ascend/W8A8_DYNAMIC] Number of global experts mismatch " + "(excluding redundancy). " + f"router_experts={router_logits.shape[1]}, " + f"expected_experts={num_logical_experts}, " + f"zero_expert_num={zero_expert_num}, " + f"zero_expert_type={zero_expert_type}" ) - if self.multistream_overlap_gate: - fc3_context = get_flash_common3_context() - assert fc3_context is not None - topk_weights = fc3_context.topk_weights - topk_ids = fc3_context.topk_ids - else: - topk_weights, topk_ids = select_experts( - hidden_states=x, - router_logits=router_logits, - top_k=top_k, - use_grouped_topk=use_grouped_topk, - renormalize=renormalize, - topk_group=topk_group, - num_expert_group=num_expert_group, - custom_routing_function=custom_routing_function, - scoring_func=scoring_func, - routed_scaling_factor=routed_scaling_factor, - e_score_correction_bias=e_score_correction_bias, - mix_placement=mix_placement, - num_logical_experts=router_logits.shape[1], - num_shared_experts=n_shared_experts, - num_experts=num_experts, - ) + topk_weights, topk_ids = select_experts( + hidden_states=x, + router_logits=router_logits, + top_k=top_k, + use_grouped_topk=use_grouped_topk, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function, + scoring_func=scoring_func, + routed_scaling_factor=routed_scaling_factor, + e_score_correction_bias=e_score_correction_bias, + mix_placement=mix_placement, + num_logical_experts=router_logits.shape[1], + num_shared_experts=n_shared_experts, + num_experts=num_logical_experts, + tid2eid=tid2eid, + ) assert topk_ids is not None assert topk_weights is not None if zero_expert_num > 0 and zero_expert_type is not None: topk_ids, topk_weights, zero_expert_result = zero_experts_compute( expert_indices=topk_ids, expert_scales=topk_weights, - num_experts=num_experts, + num_experts=num_logical_experts, zero_expert_type=zero_expert_type, hidden_states=x, ) @@ -434,22 +317,38 @@ def apply( # currently it is only activated when doing profile runs. if enable_force_load_balance: random_matrix = torch.rand( - topk_ids.size(0), num_experts, device=topk_ids.device + topk_ids.size(0), num_logical_experts, device=topk_ids.device ) topk_ids = torch.argsort(random_matrix, dim=1)[:, : topk_ids.size(1)].to( topk_ids.dtype ) # ### PATCH START: AFD force-load-balance W8A8 routing override - # Replace layer-owned profiling topk ids with deterministic balanced ids. - elif getattr(layer, "enable_force_load_balance", False): + # Replace routed ids with a deterministic balanced cycle when explicitly + # requested by the plugin configuration captured during construction. + if not enable_force_load_balance and self.enable_force_load_balance: + force_lb_config = ForceLoadBalanceConfig( + n_routed_experts=num_logical_experts, + ep_size=int(layer.moe_config.ep_size), + ep_rank=int(layer.moe_config.ep_rank), + top_k=top_k, + topn_per_rank=self.force_load_balance_topn_per_rank, + ) + if self.force_lb_fake_topk_buffer is None: + _init_force_lb_buffer( + self, + force_lb_config, + self.max_force_lb_tokens, + topk_ids.device, + ) fake_routed_topk_ids = _get_force_lb_topk_ids( - layer, - batch_tokens=topk_ids.shape[0], - device=topk_ids.device, + self, + force_lb_config, + topk_ids.shape[0], + topk_ids.device, ) fake_routed_topk_ids = fake_routed_topk_ids.to(topk_ids.dtype) - if getattr(layer, "mix_placement", False): + if mix_placement: shared_topk_ids = topk_ids[:, top_k:] topk_ids = torch.cat([fake_routed_topk_ids, shared_topk_ids], dim=1) else: @@ -459,10 +358,12 @@ def apply( assert topk_weights is not None topk_weights = topk_weights.to(self.in_dtype) + act_name = getattr(activation, "value", activation) moe_comm_method = _EXTRA_CTX.moe_comm_method fused_scale_flag = ( _EXTRA_CTX.moe_comm_type == MoECommType.FUSED_MC2 - and envs_ascend.VLLM_ASCEND_ENABLE_FUSED_MC2 == 1 + and get_ascend_config().enable_fused_mc2 == 1 + and act_name != "swigluoai_uninterleave" ) if self.dynamic_eplb: w1 = layer.w13_weight_list @@ -477,6 +378,19 @@ def apply( if fused_scale_flag else layer.w2_weight_scale_list ) + w1_scale_bias = ( + [torch.tensor([], dtype=torch.float32)] if fused_scale_flag else None + ) + w2_scale_bias = ( + [torch.tensor([], dtype=torch.float32)] if fused_scale_flag else None + ) + elif fused_scale_flag and _MEGA_MOE_SUPPORTED: + w1 = layer.cann_mega_moe_w13_weight_list + w1_scale = layer.cann_mega_moe_fused_w1_scale_list + w2 = layer.cann_mega_moe_w2_weight_list + w2_scale = layer.cann_mega_moe_fused_w2_scale_list + w1_scale_bias = None + w2_scale_bias = None else: w1 = [layer.w13_weight] w1_scale = ( @@ -488,6 +402,12 @@ def apply( w2_scale = ( [layer.fused_w2_scale] if fused_scale_flag else [layer.w2_weight_scale] ) + w1_scale_bias = ( + [torch.tensor([], dtype=torch.float32)] if fused_scale_flag else None + ) + w2_scale_bias = ( + [torch.tensor([], dtype=torch.float32)] if fused_scale_flag else None + ) final_hidden_states = moe_comm_method.fused_experts( fused_experts_input=build_fused_experts_input( @@ -507,14 +427,18 @@ def apply( activation=activation, w1_scale=w1_scale, w2_scale=w2_scale, + w1_scale_bias=w1_scale_bias, + w2_scale_bias=w2_scale_bias, + swiglu_limit=layer.swiglu_limit, + swiglu_alpha=getattr(layer, "swiglu_alpha", 1.0), + swiglu_beta=getattr(layer, "swiglu_beta", 0.0), ) ) if zero_expert_num > 0 and zero_expert_type is not None: final_hidden_states += zero_expert_result return final_hidden_states - -AscendFusedMoE.__init__ = __init__ +AscendW8A8DynamicFusedMoEMethod.__init__ = __init__ AscendW8A8DynamicFusedMoEMethod.apply = apply diff --git a/afd_plugin/model_executor/models/deepseek_v2.py b/afd_plugin/model_executor/models/deepseek_v2.py index add1a88a..aa302b37 100644 --- a/afd_plugin/model_executor/models/deepseek_v2.py +++ b/afd_plugin/model_executor/models/deepseek_v2.py @@ -17,6 +17,7 @@ from vllm.config import ParallelConfig, VllmConfig from vllm.forward_context import get_forward_context from vllm.logger import init_logger +from vllm.model_executor.layers import fused_moe from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models import deepseek_v2 as native @@ -430,6 +431,12 @@ def __init__( if self.compute_gate_on_attention and not self.is_moe_layer: self.mlp = native.PPMissingLayer() elif self.is_moe_layer: + # vLLM models bind FusedMoE at module import time. AFD can import + # native DeepSeek before vLLM-Ascend patches the package factory, + # so refresh that binding after platform initialization and before + # constructing the NPU FFN MoE. + if device_type == "npu": + native.FusedMoE = fused_moe.FusedMoE self.mlp = native.DeepseekV2MoE( config=config, parallel_config=parallel_config, diff --git a/afd_plugin/v1/worker/npu/attention_model_runner.py b/afd_plugin/v1/worker/npu/attention_model_runner.py index dcc71974..4bf3169f 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner.py @@ -32,14 +32,12 @@ set_ascend_forward_context, ) from vllm_ascend.attention.attention_v1 import AscendAttentionState -from vllm_ascend.attention.kvcomp_attn.attention_utils import build_kvcomp_metadata from vllm_ascend.attention.utils import ( AscendCommonAttentionMetadata, using_paged_attention, ) from vllm_ascend.compilation.acl_graph import ACLGraphWrapper from vllm_ascend.ops.rotary_embedding import update_cos_sin -from vllm_ascend.patch.worker.patch_module import patch_torch_npu_argsort from vllm_ascend.spec_decode.dflash_proposer import AscendDflashProposer from vllm_ascend.spec_decode.draft_proposer import AscendDraftModelProposer from vllm_ascend.spec_decode.eagle_proposer import AscendEagleProposer @@ -184,7 +182,6 @@ def _model_forward(self, *args: Any, **kwargs: Any) -> Any: self._update_full_graph_params_if_needed( forward_context, num_tokens_padded, - positions, ) hidden_states = run_model() else: @@ -192,7 +189,6 @@ def _model_forward(self, *args: Any, **kwargs: Any) -> Any: self._update_full_graph_params_if_needed( forward_context, num_tokens_padded, - positions, ) if ( @@ -573,7 +569,6 @@ def _build_attn_group_metadata( extra_attn_metadata_args = {} if use_spec_decode and isinstance(builder, GDNAttentionMetadataBuilder): assert ubid is None, "UBatching not supported with GDN yet" - patch_torch_npu_argsort() extra_attn_metadata_args = dict( num_accepted_tokens=self.num_accepted_tokens.gpu[:num_reqs_padded], num_decode_draft_tokens_cpu=self.num_decode_draft_tokens.cpu[ @@ -645,8 +640,6 @@ def _build_attn_group_metadata( spec_decode_common_attn_metadata = cm else: spec_decode_common_attn_metadata = cm - if self.enable_hamming_sparse is True: - build_kvcomp_metadata(self.kvcomp_meta_data, cm) for attn_gid in range(len(self.attn_groups[kv_cache_gid])): ubatch_common_metadata = split_attn_metadata( ubatch_slices, @@ -1394,7 +1387,6 @@ def _sync_metadata_across_dp( moe_comm_type = select_moe_comm_method( num_tokens_padded, self.vllm_config, - is_draft_model, ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, @@ -1414,7 +1406,6 @@ def _sync_metadata_across_dp( moe_comm_type = select_moe_comm_method( num_tokens_padded, self.vllm_config, - is_draft_model, ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, @@ -1448,7 +1439,6 @@ def _sync_metadata_across_dp( moe_comm_type = select_moe_comm_method( num_tokens_padded, self.vllm_config, - is_draft_model, ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, @@ -1478,7 +1468,6 @@ def _sync_metadata_across_dp( moe_comm_type = select_moe_comm_method( max_tokens_across_dp, self.vllm_config, - is_draft_model, ) should_ubatch = check_enable_ubatch( min_tokens_across_dp, diff --git a/tests/unit/compat/patches/test_force_load_balance.py b/tests/unit/compat/patches/test_force_load_balance.py index 04025e90..6662942d 100644 --- a/tests/unit/compat/patches/test_force_load_balance.py +++ b/tests/unit/compat/patches/test_force_load_balance.py @@ -3,9 +3,7 @@ import importlib import sys import types -from collections.abc import Callable from types import SimpleNamespace -from typing import Any import pytest @@ -13,17 +11,10 @@ class _QuantType: - NONE = 0 W8A8 = 1 def _install_fake_modules(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: - class AscendFusedMoE: - """Stand-in for vllm_ascend AscendFusedMoE.""" - - def __init__(self, *args: object, **kwargs: object) -> None: - del args, kwargs - def build_fused_experts_input(*args: object, **kwargs: object) -> torch.Tensor: """Fake builder: returns the possibly swapped topk_ids.""" @@ -31,88 +22,40 @@ def build_fused_experts_input(*args: object, **kwargs: object) -> torch.Tensor: return kwargs["topk_ids"] class AscendW8A8DynamicFusedMoEMethod: - def __init__(self): - self.multistream_overlap_gate = False - self.in_dtype = torch.float32 - self.dynamic_eplb = False - self.quant_type = _QuantType.W8A8 - - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - router_logits: torch.Tensor, - top_k: int, - renormalize: bool, - use_grouped_topk: bool = False, - num_experts: int = -1, - expert_map: torch.Tensor | None = None, - topk_group: int | None = None, - num_expert_group: int | None = None, - custom_routing_function: Callable | None = None, - scoring_func: str = "softmax", - routed_scaling_factor: float = 1.0, - e_score_correction_bias: torch.Tensor | None = None, - is_prefill: bool = True, - enable_force_load_balance: bool = False, - log2phy: torch.Tensor | None = None, - global_redundant_expert_num: int = 0, - pertoken_scale: Any | None = None, - activation: str = "silu", - apply_router_weight_on_input: bool = False, - mc2_mask: torch.Tensor | None = None, - ) -> torch.Tensor: - import vllm_ascend.quantization.methods.w8a8_dynamic as mod - - del ( - layer, - top_k, - renormalize, - use_grouped_topk, - num_experts, - expert_map, - topk_group, - num_expert_group, - custom_routing_function, - scoring_func, - routed_scaling_factor, - e_score_correction_bias, - is_prefill, - enable_force_load_balance, - log2phy, - global_redundant_expert_num, - pertoken_scale, - activation, - apply_router_weight_on_input, - mc2_mask, - ) - return mod.build_fused_experts_input( - hidden_states=x, - topk_weights=torch.ones_like(router_logits, dtype=torch.float32), - topk_ids=router_logits, - w1=torch.empty(0), - w2=torch.empty(0), - quant_type=_QuantType.W8A8, - dynamic_eplb=False, - ) + quant_type = _QuantType.W8A8 vllm = types.ModuleType("vllm") vllm_config = types.ModuleType("vllm.config") + vllm_config.CompilationMode = SimpleNamespace(VLLM_COMPILE="vllm_compile") vllm_config.VllmConfig = object + current_vllm_config = SimpleNamespace( + additional_config={}, + compilation_config=SimpleNamespace(mode="none"), + model_config=SimpleNamespace(enforce_eager=True, dtype=torch.float32), + scheduler_config=SimpleNamespace(max_num_batched_tokens=8), + ) + vllm_config.get_current_vllm_config = lambda: current_vllm_config + vllm_logger = types.ModuleType("vllm.logger") + vllm_logger.logger = SimpleNamespace(warning_once=lambda *args, **kwargs: None) root = types.ModuleType("vllm_ascend") - envs_mod = types.ModuleType("vllm_ascend.envs") - envs_mod.VLLM_ASCEND_ENABLE_FUSED_MC2 = 0 + ascend_config_mod = types.ModuleType("vllm_ascend.ascend_config") + ascend_config_mod.get_ascend_config = lambda: SimpleNamespace( + enable_fused_mc2=0, + eplb_config=SimpleNamespace(dynamic_eplb=False), + ) ascend_forward_context_mod = types.ModuleType("vllm_ascend.ascend_forward_context") ascend_forward_context_mod.MoECommType = SimpleNamespace(FUSED_MC2="fused_mc2") + ascend_forward_context_mod._MEGA_MOE_SUPPORTED = False ascend_forward_context_mod._EXTRA_CTX = SimpleNamespace( moe_comm_method=SimpleNamespace( fused_experts=lambda fused_experts_input: fused_experts_input ), moe_comm_type=None, ) - flash_common3_context_mod = types.ModuleType("vllm_ascend.flash_common3_context") - flash_common3_context_mod.get_flash_common3_context = lambda: None + distributed = types.ModuleType("vllm_ascend.distributed") + parallel_state_mod = types.ModuleType("vllm_ascend.distributed.parallel_state") + parallel_state_mod.get_mc2_group = lambda: SimpleNamespace() ops = types.ModuleType("vllm_ascend.ops") fused_moe_pkg = types.ModuleType("vllm_ascend.ops.fused_moe") experts_selector_mod = types.ModuleType( @@ -135,35 +78,69 @@ def select_experts( num_logical_experts, num_shared_experts, num_experts, + tid2eid, ): + del ( + hidden_states, + use_grouped_topk, + renormalize, + topk_group, + num_expert_group, + custom_routing_function, + scoring_func, + routed_scaling_factor, + e_score_correction_bias, + mix_placement, + num_logical_experts, + num_shared_experts, + num_experts, + tid2eid, + ) + topk_ids = router_logits[:, :top_k].to(torch.int64) return ( - torch.ones_like(router_logits, dtype=torch.float32), - router_logits, + torch.ones_like(topk_ids, dtype=torch.float32), + topk_ids, ) experts_selector_mod.select_experts = select_experts experts_selector_mod.zero_experts_compute = None fused_moe_mod = types.ModuleType("vllm_ascend.ops.fused_moe.fused_moe") - fused_moe_mod.AscendFusedMoE = AscendFusedMoE fused_moe_mod.logger = SimpleNamespace( info=lambda *args, **kwargs: None, warning=lambda *args, **kwargs: None, info_once=lambda *args, **kwargs: None, ) + moe_runtime_args_mod = types.ModuleType( + "vllm_ascend.ops.fused_moe.moe_runtime_args" + ) + moe_runtime_args_mod.build_fused_experts_input = build_fused_experts_input quant = types.ModuleType("vllm_ascend.quantization") methods = types.ModuleType("vllm_ascend.quantization.methods") + methods_base_mod = types.ModuleType("vllm_ascend.quantization.methods.base") + + def get_moe_num_logical_experts( + layer, + num_experts, + global_redundant_expert_num=0, + num_shared_experts=0, + ): + num_logical_experts = getattr(layer.moe_config, "num_logical_experts", None) + if num_logical_experts is not None: + return int(num_logical_experts) + return int( + num_experts - global_redundant_expert_num - num_shared_experts + ) + + methods_base_mod.get_moe_num_logical_experts = get_moe_num_logical_experts w8a8_mod = types.ModuleType("vllm_ascend.quantization.methods.w8a8_dynamic") w8a8_mod.AscendW8A8DynamicFusedMoEMethod = AscendW8A8DynamicFusedMoEMethod - w8a8_mod.build_fused_experts_input = build_fused_experts_input - - quant_type_mod = types.ModuleType("vllm_ascend.quantization.quant_type") - quant_type_mod.QuantType = _QuantType monkeypatch.setitem(sys.modules, "vllm", vllm) monkeypatch.setitem(sys.modules, "vllm.config", vllm_config) + monkeypatch.setitem(sys.modules, "vllm.logger", vllm_logger) monkeypatch.setitem(sys.modules, "vllm_ascend", root) - monkeypatch.setitem(sys.modules, "vllm_ascend.envs", envs_mod) + monkeypatch.setitem(sys.modules, "vllm_ascend.ascend_config", ascend_config_mod) monkeypatch.setitem( sys.modules, "vllm_ascend.ascend_forward_context", @@ -171,8 +148,13 @@ def select_experts( ) monkeypatch.setitem( sys.modules, - "vllm_ascend.flash_common3_context", - flash_common3_context_mod, + "vllm_ascend.distributed", + distributed, + ) + monkeypatch.setitem( + sys.modules, + "vllm_ascend.distributed.parallel_state", + parallel_state_mod, ) monkeypatch.setitem(sys.modules, "vllm_ascend.ops", ops) monkeypatch.setitem(sys.modules, "vllm_ascend.ops.fused_moe", fused_moe_pkg) @@ -184,13 +166,20 @@ def select_experts( monkeypatch.setitem( sys.modules, "vllm_ascend.ops.fused_moe.fused_moe", fused_moe_mod ) + monkeypatch.setitem( + sys.modules, + "vllm_ascend.ops.fused_moe.moe_runtime_args", + moe_runtime_args_mod, + ) monkeypatch.setitem(sys.modules, "vllm_ascend.quantization", quant) monkeypatch.setitem(sys.modules, "vllm_ascend.quantization.methods", methods) monkeypatch.setitem( - sys.modules, "vllm_ascend.quantization.methods.w8a8_dynamic", w8a8_mod + sys.modules, + "vllm_ascend.quantization.methods.base", + methods_base_mod, ) monkeypatch.setitem( - sys.modules, "vllm_ascend.quantization.quant_type", quant_type_mod + sys.modules, "vllm_ascend.quantization.methods.w8a8_dynamic", w8a8_mod ) return fused_moe_mod @@ -205,8 +194,8 @@ def force_lb_mod(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: return mod -def _new_layer(force_lb_mod: types.ModuleType) -> object: - return force_lb_mod.AscendFusedMoE.__new__(force_lb_mod.AscendFusedMoE) +def _new_layer() -> SimpleNamespace: + return SimpleNamespace() def _aggregate_target_rank_counts( @@ -241,21 +230,24 @@ def _aggregate_target_rank_counts( def test_force_load_balance_buffer_topn_per_rank(force_lb_mod: types.ModuleType): - layer = _new_layer(force_lb_mod) - layer.ep_size = 4 - layer.ep_rank = 0 - layer.n_routed_experts = 8 - layer.top_k = 2 - layer.force_load_balance_topn_per_rank = 1 + method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + config = force_lb_mod.ForceLoadBalanceConfig( + n_routed_experts=8, + ep_size=4, + ep_rank=0, + top_k=2, + topn_per_rank=1, + ) force_lb_mod._init_force_lb_buffer( - layer, + method, + config, max_tokens=4, device=torch.device("cpu"), ) expected = torch.tensor([[0, 2], [4, 6], [0, 2], [4, 6]], dtype=torch.int32) - assert torch.equal(layer.force_lb_fake_topk_buffer, expected) + assert torch.equal(method.force_lb_fake_topk_buffer, expected) def test_force_load_balance_buffer_uses_max_num_batched_tokens( @@ -266,20 +258,23 @@ def test_force_load_balance_buffer_uses_max_num_batched_tokens( ) assert max_tokens == 6 - layer = _new_layer(force_lb_mod) - layer.ep_size = 2 - layer.ep_rank = 0 - layer.n_routed_experts = 4 - layer.top_k = 2 - layer.force_load_balance_topn_per_rank = 0 + method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + config = force_lb_mod.ForceLoadBalanceConfig( + n_routed_experts=4, + ep_size=2, + ep_rank=0, + top_k=2, + topn_per_rank=0, + ) force_lb_mod._init_force_lb_buffer( - layer, + method, + config, max_tokens=max_tokens, device=torch.device("cpu"), ) - assert layer.force_lb_fake_topk_buffer.shape == (6, 2) + assert method.force_lb_fake_topk_buffer.shape == (6, 2) def test_force_load_balance_max_tokens_falls_back_when_not_int( @@ -294,21 +289,23 @@ def test_force_load_balance_max_tokens_falls_back_when_not_int( def test_force_load_balance_buffer_ids_within_routed_experts( force_lb_mod: types.ModuleType, ): - layer = _new_layer(force_lb_mod) - layer.ep_size = 2 - layer.ep_rank = 0 - layer.n_routed_experts = 4 - layer.global_num_experts = 6 - layer.top_k = 2 - layer.force_load_balance_topn_per_rank = 2 + method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + config = force_lb_mod.ForceLoadBalanceConfig( + n_routed_experts=4, + ep_size=2, + ep_rank=0, + top_k=2, + topn_per_rank=2, + ) force_lb_mod._init_force_lb_buffer( - layer, + method, + config, max_tokens=2, device=torch.device("cpu"), ) - assert int(layer.force_lb_fake_topk_buffer.max()) < layer.n_routed_experts + assert int(method.force_lb_fake_topk_buffer.max()) < config.n_routed_experts def test_force_load_balance_full_expert_cycle_is_deterministic( @@ -407,81 +404,115 @@ def test_force_load_balance_all_experts_aggregates_partial_cycle_evenly( def test_force_load_balance_buffer_grows_for_large_batch( force_lb_mod: types.ModuleType, ): - layer = _new_layer(force_lb_mod) - layer.ep_size = 2 - layer.ep_rank = 0 - layer.n_routed_experts = 4 - layer.top_k = 2 - layer.force_load_balance_topn_per_rank = 2 + method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + config = force_lb_mod.ForceLoadBalanceConfig( + n_routed_experts=4, + ep_size=2, + ep_rank=0, + top_k=2, + topn_per_rank=2, + ) force_lb_mod._init_force_lb_buffer( - layer, + method, + config, max_tokens=2, device=torch.device("cpu"), ) topk_ids = force_lb_mod._get_force_lb_topk_ids( - layer, + method, + config, batch_tokens=5, device=torch.device("cpu"), ) assert topk_ids.shape == (5, 2) - assert layer.force_lb_fake_topk_buffer.shape[0] >= 5 + assert method.force_lb_fake_topk_buffer.shape[0] >= 5 -def test_w8a8_apply_swaps_topk_ids_with_buffer(force_lb_mod: types.ModuleType): +def test_w8a8_apply_lazily_builds_and_swaps_topk_ids( + force_lb_mod: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, +): + vllm_config = force_lb_mod.get_current_vllm_config() + vllm_config.additional_config = { + "enable_force_load_balance": True, + "force_load_balance_topn_per_rank": 1, + } method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + assert method.enable_force_load_balance + assert method.force_load_balance_topn_per_rank == 1 + assert method.force_lb_fake_topk_buffer is None + monkeypatch.setattr( + force_lb_mod, + "get_current_vllm_config", + lambda: (_ for _ in ()).throw(AssertionError("outside config context")), + ) - layer = _new_layer(force_lb_mod) - layer.enable_force_load_balance = True + layer = _new_layer() layer.mix_placement = False - layer.top_k = 2 - layer.ep_size = 4 - layer.n_routed_experts = 8 - layer.force_load_balance_topn_per_rank = 1 - layer.force_lb_fake_topk_buffer = torch.tensor( - [[0, 2], [4, 6], [0, 2], [4, 6]], dtype=torch.int32 + layer.moe_config = SimpleNamespace( + ep_size=4, + ep_rank=0, + num_logical_experts=8, ) layer.w13_weight = torch.empty(0) layer.w13_weight_scale_fp32 = torch.empty(0) layer.w2_weight = torch.empty(0) layer.w2_weight_scale = torch.empty(0) + layer.swiglu_limit = None - real_topk_ids = torch.zeros((4, 2), dtype=torch.int64) + router_logits = torch.zeros((4, 8), dtype=torch.float32) out = method.apply( layer=layer, x=torch.empty((4, 1)), - router_logits=real_topk_ids, + router_logits=router_logits, top_k=2, renormalize=True, - num_experts=2, + num_experts=8, ) - expected = layer.force_lb_fake_topk_buffer.to(torch.int64) + expected = torch.tensor([[0, 2], [4, 6], [0, 2], [4, 6]]) assert torch.equal(out, expected) + assert method.force_lb_fake_topk_buffer.shape == (8, 2) + for field_name in ("n_routed_experts", "ep_size", "ep_rank", "top_k"): + assert not hasattr(layer, field_name) -def test_w8a8_apply_passthrough_when_buffer_absent(force_lb_mod: types.ModuleType): +def test_w8a8_apply_passthrough_when_plugin_disabled( + force_lb_mod: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, +): method = force_lb_mod.AscendW8A8DynamicFusedMoEMethod() + assert not method.enable_force_load_balance + monkeypatch.setattr( + force_lb_mod, + "get_current_vllm_config", + lambda: (_ for _ in ()).throw(AssertionError("outside config context")), + ) - layer = _new_layer(force_lb_mod) - layer.enable_force_load_balance = False - layer.force_lb_fake_topk_buffer = None + layer = _new_layer() layer.mix_placement = False - layer.top_k = 2 + layer.moe_config = SimpleNamespace( + ep_size=1, + ep_rank=0, + num_logical_experts=2, + ) layer.w13_weight = torch.empty(0) layer.w13_weight_scale_fp32 = torch.empty(0) layer.w2_weight = torch.empty(0) layer.w2_weight_scale = torch.empty(0) + layer.swiglu_limit = None - real_topk_ids = torch.zeros((4, 2), dtype=torch.int64) + router_logits = torch.arange(8, dtype=torch.float32).reshape(4, 2) out = method.apply( layer=layer, x=torch.empty((4, 1)), - router_logits=real_topk_ids, + router_logits=router_logits, top_k=2, renormalize=True, num_experts=2, ) - assert torch.equal(out, real_topk_ids) + assert torch.equal(out, router_logits.to(torch.int64)) + assert method.force_lb_fake_topk_buffer is None diff --git a/tests/unit/model_executor/models/test_deepseek_v2_construction.py b/tests/unit/model_executor/models/test_deepseek_v2_construction.py index 82ae52d7..21733bef 100644 --- a/tests/unit/model_executor/models/test_deepseek_v2_construction.py +++ b/tests/unit/model_executor/models/test_deepseek_v2_construction.py @@ -405,6 +405,28 @@ def test_ffn_constructs_no_real_attention( assert not any(name.startswith("self_attn.") for name in _parameter_names(moe)) +def test_npu_ffn_refreshes_native_fused_moe_factory( + monkeypatch, + construction_env, +): + stale_factory = object() + ascend_factory = object() + factories_seen_by_native_moe = [] + + class _FakeMoE(nn.Module): + def __init__(self, **_kwargs): + super().__init__() + factories_seen_by_native_moe.append(adapter.native.FusedMoE) + + monkeypatch.setattr(adapter.native, "FusedMoE", stale_factory) + monkeypatch.setattr(adapter.fused_moe, "FusedMoE", ascend_factory) + monkeypatch.setattr(adapter.native, "DeepseekV2MoE", _FakeMoE) + + _make_layer(monkeypatch, role="ffn", layer_idx=1) + + assert factories_seen_by_native_moe == [ascend_factory] + + @pytest.mark.parametrize( ("aiter_enabled", "apply_routed_scale_to_output"), [(False, True), (True, False)], From 37013a321d982f6ecfd8c97533153a3bd3652827 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Sun, 2 Aug 2026 11:19:17 +0800 Subject: [PATCH 03/10] refactor(npu): remove PCP support from model runner v1 Signed-off-by: jiangkuaixue123 --- .../v1/worker/npu/attention_model_runner.py | 199 +------------ afd_plugin/v1/worker/npu/pcp_debug.py | 277 ------------------ docs/design/module/execution_platforms.md | 1 - docs/design/module/index.md | 2 +- tests/unit/v1/worker/test_npu_runtime.py | 12 - 5 files changed, 5 insertions(+), 486 deletions(-) delete mode 100644 afd_plugin/v1/worker/npu/pcp_debug.py diff --git a/afd_plugin/v1/worker/npu/attention_model_runner.py b/afd_plugin/v1/worker/npu/attention_model_runner.py index 4bf3169f..f297d981 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner.py @@ -26,7 +26,6 @@ from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec -from vllm.v1.worker.ubatch_utils import UBatchSlice from vllm_ascend.ascend_forward_context import ( select_moe_comm_method, set_ascend_forward_context, @@ -76,17 +75,6 @@ _with_dp_derived_afd_rank, ) from afd_plugin.v1.worker.npu.npu_ubatch_wrapper import AscendUBatchWrapper -from afd_plugin.v1.worker.npu.pcp_debug import ( - clone_pcp_metadata, - debug_pcp_common_metadata_summary, - debug_pcp_manager_summary, - debug_pcp_metadata_enabled, - debug_pcp_metadata_summary, - debug_slice_summary, - debug_value_summary, - restore_pcp_manager_state, - snapshot_pcp_manager_state, -) from afd_plugin.v1.worker.npu.ubatch_utils import ( check_enable_ubatch, create_request_boundary_ubatch_slices, @@ -327,157 +315,9 @@ def _build_attention_metadata_with_ubatches( kv_cache_groups = self.kv_cache_config.kv_cache_groups - def _get_pcp_metadata(block_table_tensor): - if not self.use_cp: - return None, block_table_tensor - return self.pcp_manager.generate_pcp_metadata( - num_tokens, - self.query_lens, - self.input_batch, - num_scheduled_tokens_np, - block_table_tensor, - num_reqs_padded, - num_reqs, - ) - - def _build_stage_local_pcp_metadata( - common_attn_metadata: AscendCommonAttentionMetadata, - ubatch_slice: UBatchSlice, - ubid: int, - kv_cache_gid: int, - attn_gid: int, - ) -> None: - if not self.use_cp or int(self.pcp_size) <= 1: - return - if self.speculative_config is not None: - raise RuntimeError( - "async_moe_ubatching with PCP does not support speculative " - "decode metadata yet", - ) - if bool(getattr(self.pcp_manager, "pcp_use_hybrid_attn", False)): - raise RuntimeError( - "async_moe_ubatching with PCP does not support hybrid " - "attention metadata yet", - ) - - full_num_scheduled_tokens = ( - self.pcp_manager.query_lens_pcp_full.cpu[:num_reqs] - .to("cpu") - .numpy() - .copy() - ) - original_num_scheduled_tokens = full_num_scheduled_tokens[ - ubatch_slice.request_slice - ].copy() - original_token_start = int( - full_num_scheduled_tokens[: ubatch_slice.request_slice.start].sum(), - ) - original_token_stop = int( - full_num_scheduled_tokens[: ubatch_slice.request_slice.stop].sum(), - ) - original_common_summary = debug_pcp_common_metadata_summary( - common_attn_metadata, - ) - stage_num_reqs = ubatch_slice.request_slice.stop - ( - ubatch_slice.request_slice.start - ) - manager_state = snapshot_pcp_manager_state(self.pcp_manager) - try: - self.pcp_manager.init_batch_info( - original_num_scheduled_tokens, - stage_num_reqs, - ) - stage_pcp_tokens, _ = self.pcp_manager.update_tokens_for_pcp( - original_num_scheduled_tokens, - self.arange_np, - ) - stage_query_lens = torch.from_numpy(stage_pcp_tokens).to( - self.query_lens.device, - ) - if debug_pcp_metadata_enabled(): - logger.warning( - "AFD PCP stage split input; kv_cache_gid=%s attn_gid=%s " - "ubid=%s request_slice=%s token_slice=%s " - "stage_num_reqs=%s original_num_scheduled_tokens=%s " - "stage_pcp_tokens=%s common=%s manager_before=%s", - kv_cache_gid, - attn_gid, - ubid, - debug_slice_summary(ubatch_slice.request_slice), - debug_slice_summary(ubatch_slice.token_slice), - stage_num_reqs, - debug_value_summary(original_num_scheduled_tokens), - debug_value_summary(stage_pcp_tokens), - original_common_summary, - debug_pcp_manager_summary(self.pcp_manager), - ) - pcp_metadata, block_table_tensor = ( - self.pcp_manager.generate_pcp_metadata( - int(common_attn_metadata.num_actual_tokens), - stage_query_lens, - self.input_batch, - stage_pcp_tokens, - common_attn_metadata.block_table_tensor, - stage_num_reqs, - stage_num_reqs, - ) - ) - if original_token_stop > original_token_start: - raw_slot_mapping = self.input_batch.block_table[ - kv_cache_gid - ].slot_mapping.gpu[original_token_start:original_token_stop] - stage_num_tokens = int(stage_pcp_tokens.sum()) - common_attn_metadata.slot_mapping = ( - self.pcp_manager.get_padded_slot_mapping( - stage_num_tokens, - stage_num_tokens, - raw_slot_mapping, - kv_cache_gid, - ).clone() - ) - common_attn_metadata.prefill_context_parallel_metadata = ( - clone_pcp_metadata(pcp_metadata) - ) - common_attn_metadata.block_table_tensor = block_table_tensor - if debug_pcp_metadata_enabled(): - logger.warning( - "AFD PCP stage split result; kv_cache_gid=%s attn_gid=%s " - "ubid=%s request_slice=%s token_slice=%s " - "pcp_metadata=%s block_table_tensor=%s common_after=%s " - "manager_after=%s", - kv_cache_gid, - attn_gid, - ubid, - debug_slice_summary(ubatch_slice.request_slice), - debug_slice_summary(ubatch_slice.token_slice), - debug_pcp_metadata_summary(pcp_metadata), - debug_value_summary(block_table_tensor), - debug_pcp_common_metadata_summary(common_attn_metadata), - debug_pcp_manager_summary(self.pcp_manager), - ) - finally: - restore_pcp_manager_state(self.pcp_manager, manager_state) - def _get_block_table_and_slot_mapping(kv_cache_gid: int): assert num_reqs_padded is not None and num_tokens_padded is not None kv_cache_spec = kv_cache_groups[kv_cache_gid].kv_cache_spec - if self.pcp_size > 1: - total_num_pcp_pads = sum(self.pcp_manager.num_pcp_pads_cpu[:num_reqs]) - if self.pcp_manager.pcp_use_hybrid_attn: - num_scheduled_tokens_padded = ( - self.pcp_manager.num_scheduled_tokens_padded - ) - assert num_scheduled_tokens_padded is not None - maybe_pcp_full_tokens = ( - sum(num_scheduled_tokens_padded) * self.pcp_size - - total_num_pcp_pads - ) - else: - maybe_pcp_full_tokens = ( - num_tokens * self.pcp_size - total_num_pcp_pads - ) - else: - maybe_pcp_full_tokens = num_tokens_padded if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec): blk_table_tensor = torch.zeros( (num_reqs_padded, 1), @@ -491,31 +331,15 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): ) else: blk_table = self.input_batch.block_table[kv_cache_gid] - slot_mapping = blk_table.slot_mapping.gpu[:maybe_pcp_full_tokens] - maybe_num_reqs_padded = ( - num_reqs_padded * self.decode_token_per_req - if self.use_cp - else num_reqs_padded - ) - blk_table_tensor = blk_table.get_device_tensor()[:maybe_num_reqs_padded] - if self.pcp_size == 1: - slot_mapping[num_tokens:num_tokens_padded].fill_(-1) - blk_table_tensor[num_reqs:num_reqs_padded].fill_(0) - if self.pcp_size > 1: - slot_mapping = self.pcp_manager.get_padded_slot_mapping( - num_tokens, - num_tokens_padded, - slot_mapping, - kv_cache_gid, - ) + slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded] + blk_table_tensor = blk_table.get_device_tensor()[:num_reqs_padded] + slot_mapping[num_tokens:num_tokens_padded].fill_(-1) + blk_table_tensor[num_reqs:num_reqs_padded].fill_(0) if self.model_config.enable_return_routed_experts and kv_cache_gid == 0: self.cpu_slot_mapping = slot_mapping.cpu().numpy() return blk_table_tensor, slot_mapping block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0) - self.long_seq_metadata, block_table_gid_0 = _get_pcp_metadata( - block_table_gid_0, - ) num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[ :num_reqs_padded ] @@ -543,7 +367,6 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): positions=self.positions, attn_state=self.attn_state, decode_token_per_req=self.decode_token_per_req, - prefill_context_parallel_metadata=self.long_seq_metadata, ) if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill: @@ -647,13 +470,6 @@ def _build_attn_group_metadata( num_tokens_padded, ) for ubid, ubatch_cm in enumerate(ubatch_common_metadata): - _build_stage_local_pcp_metadata( - ubatch_cm, - ubatch_slices[ubid], - ubid, - kv_cache_gid, - attn_gid, - ) _build_attn_group_metadata(kv_cache_gid, attn_gid, ubatch_cm, ubid) if self.is_mm_prefix_lm: @@ -974,13 +790,6 @@ def _dummy_run_with_ubatches( force_has_lora=num_active_loras > 0, force_num_active_loras=num_active_loras, ) - if self.use_cp: - self.pcp_manager.init_batch_info(num_scheduled_tokens, num_reqs) - if self.speculative_config: - self.pcp_manager.query_lens_pcp_full.cpu[:num_reqs] = torch.from_numpy( - num_scheduled_tokens, - ) - self.pcp_manager.query_lens_pcp_full.copy_to_gpu() if cudagraph_runtime_mode is None: cudagraph_runtime_mode = _cudagraph_mode else: diff --git a/afd_plugin/v1/worker/npu/pcp_debug.py b/afd_plugin/v1/worker/npu/pcp_debug.py deleted file mode 100644 index fa0cbf6f..00000000 --- a/afd_plugin/v1/worker/npu/pcp_debug.py +++ /dev/null @@ -1,277 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Debug helpers for Ascend PCP metadata handling.""" - -from __future__ import annotations - -import copy -import os -from typing import Any - - -def debug_pcp_metadata_enabled() -> bool: - return os.getenv("AFD_DEBUG_PCP_METADATA", "0").lower() in { - "1", - "true", - "yes", - "on", - } - - -def debug_slice_summary(value: Any) -> tuple[Any, Any, Any]: - return ( - getattr(value, "start", None), - getattr(value, "stop", None), - getattr(value, "step", None), - ) - - -def debug_scalar(value: Any) -> Any: - try: - if hasattr(value, "item"): - return value.item() - return int(value) - except Exception: - return repr(value) - - -def debug_value_summary(value: Any, *, limit: int = 8) -> Any: - if value is None: - return None - summary: dict[str, Any] = {"type": type(value).__name__} - if hasattr(value, "shape"): - try: - summary["shape"] = tuple(int(dim) for dim in value.shape) - except Exception: - summary["shape"] = repr(value.shape) - if hasattr(value, "dtype"): - summary["dtype"] = str(value.dtype) - if hasattr(value, "device"): - summary["device"] = str(value.device) - - try: - flat = value.detach().flatten().to("cpu") if hasattr(value, "detach") else None - if flat is not None: - total = int(flat.numel()) - summary["numel"] = total - head_count = min(limit, total) - summary["head"] = [debug_scalar(item) for item in flat[:head_count]] - if total > limit: - tail_count = min(limit, total) - summary["tail"] = [debug_scalar(item) for item in flat[-tail_count:]] - return summary - except Exception as exc: - summary["values_error"] = repr(exc) - return summary - - try: - if hasattr(value, "reshape") and hasattr(value, "size"): - flat = value.reshape(-1) - total = int(flat.size) - summary["size"] = total - head_count = min(limit, total) - summary["head"] = [debug_scalar(item) for item in flat[:head_count]] - if total > limit: - summary["tail"] = [debug_scalar(item) for item in flat[-limit:]] - return summary - except Exception as exc: - summary["values_error"] = repr(exc) - return summary - - if isinstance(value, (list, tuple)): - total = len(value) - summary["len"] = total - summary["head"] = [debug_scalar(item) for item in value[:limit]] - if total > limit: - summary["tail"] = [debug_scalar(item) for item in value[-limit:]] - return summary - - if isinstance(value, dict): - summary["len"] = len(value) - summary["keys"] = list(value.keys())[:limit] - return summary - - return value - - -def debug_pcp_metadata_summary(pcp_metadata: Any) -> Any: - if pcp_metadata is None: - return None - fields = ( - "query_start_loc", - "query_start_loc_cpu", - "seq_lens", - "seq_lens_cpu", - "num_computed_tokens_cpu", - "q_head_idx_tensor", - "q_tail_idx_tensor", - "q_full_idx", - "pcp_allgather_restore_idx", - "pcp_unpad_mask", - "pcp_fa_query_idx", - "pcp_enter_fa_restore_idx", - "pcp_exit_fa_scatter_idx", - "num_computed_tokens_of_pcp_dcp", - "query_lens_pcp_full_cpu", - "num_actual_tokens_pcp_padded", - "actual_seq_lengths_q", - "actual_seq_lengths_kv", - "actual_seq_lengths_query", - "actual_seq_lengths_key", - "slot_mapping_cp", - ) - summary: dict[str, Any] = {"type": type(pcp_metadata).__name__} - for name in fields: - if hasattr(pcp_metadata, name): - summary[name] = debug_value_summary(getattr(pcp_metadata, name)) - return summary - - -def debug_pcp_common_metadata_summary(common_attn_metadata: Any) -> dict[str, Any]: - fields = ( - "num_reqs", - "num_actual_tokens", - "num_input_tokens", - "max_query_len", - "max_seq_len", - "query_start_loc", - "query_start_loc_cpu", - "seq_lens", - "seq_lens_cpu", - "num_computed_tokens_cpu", - "block_table_tensor", - "slot_mapping", - ) - summary: dict[str, Any] = {"type": type(common_attn_metadata).__name__} - for name in fields: - if hasattr(common_attn_metadata, name): - summary[name] = debug_value_summary(getattr(common_attn_metadata, name)) - if hasattr(common_attn_metadata, "prefill_context_parallel_metadata"): - summary["prefill_context_parallel_metadata"] = debug_pcp_metadata_summary( - common_attn_metadata.prefill_context_parallel_metadata, - ) - return summary - - -def debug_pcp_manager_summary(pcp_manager: Any) -> dict[str, Any]: - fields = ( - "num_reqs", - "num_decode_reqs", - "num_prefill_reqs", - "num_decode_tokens", - "num_scheduled_tokens_padded", - "pcp_padded_tokens_length", - "pcp_padded_tokens_fla", - "num_actual_tokens_pcp_padded", - "total_num_sampled_tokens_pcp", - "pcp_tokens", - "pcp_tokens_padded", - "num_pcp_pads_cpu", - "pcp_unpad_mask_cpu", - "max_num_tokens_across_pcp", - "total_num_scheduled_tokens", - "total_pcp_padding_tokens_fla", - "q_head_idx_tensor", - "q_tail_idx_tensor", - "q_full_idx", - ) - summary: dict[str, Any] = {"type": type(pcp_manager).__name__} - for name in fields: - if hasattr(pcp_manager, name): - summary[name] = debug_value_summary(getattr(pcp_manager, name)) - if hasattr(pcp_manager, "query_lens_pcp_full"): - query_lens = pcp_manager.query_lens_pcp_full - summary["query_lens_pcp_full.cpu"] = debug_value_summary( - getattr(query_lens, "cpu", None), - ) - summary["query_lens_pcp_full.gpu"] = debug_value_summary( - getattr(query_lens, "gpu", None), - ) - if hasattr(pcp_manager, "pcp_allgather_restore_idx"): - restore_idx = pcp_manager.pcp_allgather_restore_idx - summary["pcp_allgather_restore_idx.np"] = debug_value_summary( - getattr(restore_idx, "np", None), - ) - summary["pcp_allgather_restore_idx.gpu"] = debug_value_summary( - getattr(restore_idx, "gpu", None), - ) - return summary - - -def snapshot_pcp_manager_state(pcp_manager: Any) -> dict[str, Any]: - state: dict[str, Any] = {} - for name in ( - "num_reqs", - "num_decode_reqs", - "num_prefill_reqs", - "num_decode_tokens", - "num_scheduled_tokens_padded", - "pcp_padded_tokens_length", - "pcp_padded_tokens_fla", - "num_actual_tokens_pcp_padded", - "total_num_sampled_tokens_pcp", - "pcp_tokens_padded", - "max_num_tokens_across_pcp", - "total_num_scheduled_tokens", - "total_pcp_padding_tokens_fla", - "q_head_idx_tensor", - "q_tail_idx_tensor", - "q_full_idx", - "kv_idx_names", - "extra_long_seq_kwargs", - "long_seq_metadata", - ): - if hasattr(pcp_manager, name): - state[name] = copy.copy(getattr(pcp_manager, name)) - for name in ("pcp_tokens", "num_pcp_pads_cpu", "pcp_unpad_mask_cpu"): - if hasattr(pcp_manager, name): - state[name] = getattr(pcp_manager, name).copy() - if hasattr(pcp_manager, "query_lens_pcp_full"): - state["query_lens_pcp_full_cpu"] = pcp_manager.query_lens_pcp_full.cpu.clone() - if hasattr(pcp_manager, "pcp_allgather_restore_idx"): - restore_idx = pcp_manager.pcp_allgather_restore_idx - state["pcp_allgather_restore_idx_np"] = restore_idx.np.copy() - state["pcp_allgather_restore_idx_gpu"] = restore_idx.gpu.clone() - return state - - -def restore_pcp_manager_state(pcp_manager: Any, state: dict[str, Any]) -> None: - for name, value in state.items(): - if name == "query_lens_pcp_full_cpu": - pcp_manager.query_lens_pcp_full.cpu.copy_(value) - pcp_manager.query_lens_pcp_full.copy_to_gpu() - continue - if name == "pcp_allgather_restore_idx_np": - pcp_manager.pcp_allgather_restore_idx.np[...] = value - continue - if name == "pcp_allgather_restore_idx_gpu": - pcp_manager.pcp_allgather_restore_idx.gpu.copy_(value) - continue - if name in ("pcp_tokens", "num_pcp_pads_cpu", "pcp_unpad_mask_cpu"): - getattr(pcp_manager, name)[...] = value - continue - setattr(pcp_manager, name, value) - - -def clone_pcp_metadata(pcp_metadata: Any) -> Any: - if pcp_metadata is None: - return None - cloned = copy.copy(pcp_metadata) - attrs: dict[str, Any] = {} - if hasattr(pcp_metadata, "__dict__"): - attrs.update(vars(pcp_metadata)) - slots = getattr(pcp_metadata, "__slots__", ()) - if isinstance(slots, str): - slots = (slots,) - for slot in slots: - if hasattr(pcp_metadata, slot): - attrs.setdefault(slot, getattr(pcp_metadata, slot)) - - for name, value in attrs.items(): - if hasattr(value, "clone"): - setattr(cloned, name, value.clone()) - elif isinstance(value, list): - setattr(cloned, name, list(value)) - else: - setattr(cloned, name, copy.copy(value)) - return cloned diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index 9203dfda..aa058d63 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -14,7 +14,6 @@ primary_code_paths: - "afd_plugin/v1/worker/dbo.py" - "afd_plugin/v1/worker/npu/forward_context.py" - "afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py" - - "afd_plugin/v1/worker/npu/pcp_debug.py" - "afd_plugin/v1/worker/npu/ubatch_utils.py" - "afd_plugin/v1/worker/npu/ubatching.py" - "csrc/**" diff --git a/docs/design/module/index.md b/docs/design/module/index.md index 3c05147e..441040c6 100644 --- a/docs/design/module/index.md +++ b/docs/design/module/index.md @@ -95,7 +95,7 @@ contract. File-level entries deliberately resolve mixed directories. | [FFN runtime](ffn_runtime.md) | `afd_plugin/v1/worker/ffn_model_runner.py`, `afd_plugin/v1/worker/ffn_worker.py`, `afd_plugin/v1/worker/npu/ffn_model_runner.py`, `afd_plugin/v1/worker/npu/ffn_worker.py` | | [Connector contracts](connector_contracts.md) | `afd_plugin/connectors/**/*.py`, `afd_plugin/connectors/npu/bin/**`, `afd_plugin/distributed/**/*.py` | | [Model integration](model_integration.md) | `afd_plugin/model_executor/**/*.py` | -| [Execution platforms](execution_platforms.md) | `afd_plugin/compat/profiler.py`, `afd_plugin/compat/npu/forward_context.py`, `afd_plugin/compat/npu/ops.py`, `afd_plugin/compat/npu/profiler.py`, `afd_plugin/v1/worker/cuda_graph.py`, `afd_plugin/v1/worker/dbo.py`, `afd_plugin/v1/worker/npu/forward_context.py`, `afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py`, `afd_plugin/v1/worker/npu/pcp_debug.py`, `afd_plugin/v1/worker/npu/ubatch_utils.py`, `afd_plugin/v1/worker/npu/ubatching.py`, `csrc/**`, `setup.py`, `MANIFEST.in` | +| [Execution platforms](execution_platforms.md) | `afd_plugin/compat/profiler.py`, `afd_plugin/compat/npu/forward_context.py`, `afd_plugin/compat/npu/ops.py`, `afd_plugin/compat/npu/profiler.py`, `afd_plugin/v1/worker/cuda_graph.py`, `afd_plugin/v1/worker/dbo.py`, `afd_plugin/v1/worker/npu/forward_context.py`, `afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py`, `afd_plugin/v1/worker/npu/ubatch_utils.py`, `afd_plugin/v1/worker/npu/ubatching.py`, `csrc/**`, `setup.py`, `MANIFEST.in` | | [Compatibility and patches](compatibility_and_patches.md) | `afd_plugin/compat/__init__.py`, `afd_plugin/compat/vllm.py`, `afd_plugin/compat/npu/__init__.py`, `afd_plugin/compat/npu/feature_validation.py`, `afd_plugin/compat/npu/runtime.py`, `afd_plugin/compat/npu/runtime_config.py`, `afd_plugin/compat/patches/**/*.py` | The routing inventory covers runtime and package code under `afd_plugin/**`, diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index ad323fd2..df418115 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -1228,18 +1228,6 @@ def test_npu_async_moe_ubatching_validation_requires_supported_shape(): ), ) - fail_if_unsupported_npu_afd_features( - _vllm_config( - connector="CAMAsyncAFDConnector", - async_dp=True, - compute_gate_on_attention=True, - prefill_context_parallel_size=2, - extra_config={ - "async_moe_ubatching": True, - }, - ), - ) - with pytest.raises(RuntimeError, match="decode context parallel"): fail_if_unsupported_npu_afd_features( _vllm_config( From f585ecce7cee8fea8b342235afe5a145e775503d Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Sun, 2 Aug 2026 15:41:13 +0800 Subject: [PATCH 04/10] fix(npu): restore DBO compatibility for vLLM 0.26 Signed-off-by: jiangkuaixue123 --- afd_plugin/compat/npu/runtime.py | 4 +- .../compat/patches/config_validation.py | 103 ++- .../compat/patches/npu/ascend_platform.py | 85 +- .../v1/worker/npu/attention_model_runner.py | 860 +++++++++++------- afd_plugin/v1/worker/npu/forward_context.py | 28 +- .../v1/worker/npu/npu_ubatch_wrapper.py | 99 +- afd_plugin/v1/worker/npu/ubatch_utils.py | 76 +- .../compat/patches/test_config_validation.py | 40 +- tests/unit/compat/test_runtime.py | 26 +- tests/unit/v1/worker/test_npu_runtime.py | 177 +++- 10 files changed, 1033 insertions(+), 465 deletions(-) diff --git a/afd_plugin/compat/npu/runtime.py b/afd_plugin/compat/npu/runtime.py index c71ab7ec..e360096e 100644 --- a/afd_plugin/compat/npu/runtime.py +++ b/afd_plugin/compat/npu/runtime.py @@ -29,8 +29,8 @@ def apply_afd_ascend_patches_if_needed() -> None: apply_afd_ascend_dbo_config_patch, ) - apply_afd_ascend_dbo_config_patch() - _PATCHES_APPLIED = True + if apply_afd_ascend_dbo_config_patch(): + _PATCHES_APPLIED = True __all__ = [ diff --git a/afd_plugin/compat/patches/config_validation.py b/afd_plugin/compat/patches/config_validation.py index 529d35f0..0c4b9063 100644 --- a/afd_plugin/compat/patches/config_validation.py +++ b/afd_plugin/compat/patches/config_validation.py @@ -49,74 +49,78 @@ def create_engine_config( """Create the VllmConfig.""" assert _original_create_engine_config is not None - if not _should_relax_engine_args_backend(self): - return _original_create_engine_config( - self, - usage_context, - headless, - ) + # ### PATCH START: AFD automatic worker selection + worker_cls_was_auto = _uses_auto_worker_value(self.worker_cls) + # ### PATCH END: AFD automatic worker selection + # ### PATCH START: AFD Ascend config patch ordering + if parse_optional_afd_config(self.additional_config) is not None: + from vllm.platforms import current_platform - # ### PATCH START: AFD ubatching all2all backend validation - # vLLM validates native ubatching against DeepEP backends. AFD ubatching - # uses plugin connectors, so temporarily present a supported backend only - # while upstream builds and validates VllmConfig. - original_backend = self.all2all_backend - self.all2all_backend = _AFD_TEMP_BACKEND - try: + if current_platform.device_type == "npu": + from afd_plugin.compat.npu import apply_afd_ascend_patches_if_needed + + apply_afd_ascend_patches_if_needed() + # ### PATCH END: AFD Ascend config patch ordering + if not _should_relax_engine_args_backend(self): config = _original_create_engine_config( self, usage_context, headless, ) - finally: - self.all2all_backend = original_backend - config.parallel_config.all2all_backend = original_backend - # ### PATCH END: AFD ubatching all2all backend validation + else: + # ### PATCH START: AFD ubatching all2all backend validation + # vLLM validates native ubatching against DeepEP backends. AFD ubatching + # uses plugin connectors, so temporarily present a supported backend while + # upstream builds and validates VllmConfig. The Ascend platform wrapper + # preserves this temporary value across its default-worker normalization. + original_backend = self.all2all_backend + self.all2all_backend = _AFD_TEMP_BACKEND + try: + config = _original_create_engine_config( + self, + usage_context, + headless, + ) + finally: + self.all2all_backend = original_backend + config.parallel_config.all2all_backend = original_backend + # ### PATCH END: AFD ubatching all2all backend validation + + # ### PATCH START: AFD automatic worker selection + if worker_cls_was_auto: + _select_afd_worker_for_auto(config) + # ### PATCH END: AFD automatic worker selection return config -# Patch reason: VllmConfig validation can rerun the native ubatching all2all -# backend assertion after EngineArgs construction, and upstream auto-selects a -# platform worker that does not contain AFD role behavior. -# Patch functionality: temporarily relaxes the backend assertion for AFD -# configs, restores the real backend, then replaces an auto-selected platform -# worker with the platform- and role-specific AFD worker. +# Patch reason: EngineCore handshakes explicitly rerun VllmConfig.__post_init__ +# after the config's actual AFD all2all backend has been restored. +# Patch functionality: temporarily presents a validation-safe backend during +# explicit AFD ubatching revalidation, then restores the actual backend. # Expansion exception: upstream VllmConfig.__post_init__ is a large validation -# pipeline; keep a narrow original-function delegation so this patch only owns -# AFD validation and worker normalization. +# pipeline; keep narrow original-function delegation so this patch only owns +# the AFD backend validation bypass. # Signature: matches upstream; no added parameters. def __post_init__(self): """Verify configs are valid & consistent with each other.""" assert _original_vllm_config_post_init is not None - # ### PATCH START: AFD automatic worker selection - worker_cls_was_auto = _uses_auto_worker(self) - # ### PATCH END: AFD automatic worker selection if not _should_relax_vllm_config_backend(self): - result = _original_vllm_config_post_init(self) - else: - # ### PATCH START: AFD ubatching all2all backend validation - # Repeated VllmConfig validation can run after EngineArgs construction. - # Keep AFD's real all2all backend on the config, but use a temporary DeepEP - # value while upstream performs its native ubatching assertion. - parallel_config = self.parallel_config - original_backend = parallel_config.all2all_backend - parallel_config.all2all_backend = _AFD_TEMP_BACKEND - try: - result = _original_vllm_config_post_init(self) - finally: - parallel_config.all2all_backend = original_backend - # ### PATCH END: AFD ubatching all2all backend validation + return _original_vllm_config_post_init(self) - # ### PATCH START: AFD automatic worker selection - if worker_cls_was_auto: - _select_afd_worker_for_auto(self) - # ### PATCH END: AFD automatic worker selection + # ### PATCH START: AFD repeated ubatching backend validation + parallel_config = self.parallel_config + original_backend = parallel_config.all2all_backend + parallel_config.all2all_backend = _AFD_TEMP_BACKEND + try: + result = _original_vllm_config_post_init(self) + finally: + parallel_config.all2all_backend = original_backend + # ### PATCH END: AFD repeated ubatching backend validation return result -def _uses_auto_worker(vllm_config: VllmConfig) -> bool: - worker_cls = vllm_config.parallel_config.worker_cls +def _uses_auto_worker_value(worker_cls: str | type[Any]) -> bool: return isinstance(worker_cls, str) and worker_cls.strip() == "auto" @@ -161,8 +165,7 @@ def _should_relax_engine_args_backend(engine_args: EngineArgs) -> bool: def _should_relax_vllm_config_backend(vllm_config: VllmConfig) -> bool: if not _is_target_vllm_compatible(): return False - afd_config = parse_optional_afd_config(vllm_config) - if afd_config is None: + if parse_optional_afd_config(vllm_config) is None: return False parallel_config = vllm_config.parallel_config diff --git a/afd_plugin/compat/patches/npu/ascend_platform.py b/afd_plugin/compat/patches/npu/ascend_platform.py index d501da4a..66219994 100644 --- a/afd_plugin/compat/patches/npu/ascend_platform.py +++ b/afd_plugin/compat/patches/npu/ascend_platform.py @@ -2,12 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """Patch vLLM-Ascend platform config normalization for AFD-owned DBO. -Upstream source: ``vllm_ascend/platform.py``. +Upstream source: ``vllm_ascend/platform.py`` at commit ``80d8c194f``. """ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING from afd_plugin.config import parse_optional_afd_config @@ -17,83 +18,95 @@ _ASCEND_PLATFORM_PATCH_ATTR = "_afd_plugin_ascend_platform_patch_state" -def apply_afd_ascend_dbo_config_patch() -> None: +@dataclass(frozen=True) +class _AFDDBOConfigSnapshot: + enable_dbo: bool + ubatch_size: int + all2all_backend: str + + +def apply_afd_ascend_dbo_config_patch() -> bool: """Preserve AFD-owned DBO settings during vLLM-Ascend config normalization. vLLM-Ascend's platform compatibility pass disables DBO/ubatching fields for ordinary NPU runs. AFD owns its NPU ubatching path, so this patch snapshots those fields for AFD-enabled configs, lets upstream normalization run, then restores the AFD DBO values. The patch is a no-op when vLLM-Ascend is not - importable or when this process has already installed the wrapper. + importable. Returns whether this process has installed the wrapper (or had + already installed it), so callers do not cache a failed early import during + plugin initialization. """ try: from vllm_ascend.platform import NPUPlatform - except Exception: - return + except ImportError: + return False if hasattr(NPUPlatform, _ASCEND_PLATFORM_PATCH_ATTR): - return + return True - original_fix_incompatible_config = NPUPlatform._fix_incompatible_config + original_check_and_update_config = NPUPlatform.check_and_update_config - # Patch reason: vLLM-Ascend resets DBO fields inside NPUPlatform config - # normalization, while AFD now owns the Ascend DBO/ubatching path. + # Patch reason: vLLM-Ascend resets DBO fields in _fix_incompatible_config and + # later rewrites all2all_backend in check_and_update_config, while AFD owns + # the Ascend DBO/ubatching path and temporarily supplies a validation-safe + # backend. # Patch functionality: preserves upstream normalization for non-AFD configs and - # restores AFD DBO fields after upstream normalization for AFD-enabled configs. - # Expansion exception: upstream _fix_incompatible_config is platform-owned + # restores AFD DBO fields plus the temporary ubatching backend after upstream + # normalization for AFD-enabled configs. + # Expansion exception: upstream check_and_update_config is platform-owned # normalization; keep narrow original-function delegation so this patch only # owns the AFD DBO preservation. # Signature: matches upstream; no added parameters. - def _fix_incompatible_config(vllm_config: VllmConfig) -> Any: + def check_and_update_config(cls, vllm_config: VllmConfig) -> None: + del cls # ### PATCH START: AFD DBO config preservation saved = _snapshot_afd_dbo_config(vllm_config) + try: + original_check_and_update_config(vllm_config) + finally: + if saved is not None: + _restore_afd_dbo_config(vllm_config, saved) # ### PATCH END: AFD DBO config preservation - result = original_fix_incompatible_config(vllm_config) - # ### PATCH START: AFD DBO config preservation - if saved is not None: - _restore_afd_dbo_config(vllm_config, saved) - # ### PATCH END: AFD DBO config preservation - return result - NPUPlatform._fix_incompatible_config = staticmethod(_fix_incompatible_config) + NPUPlatform.check_and_update_config = classmethod(check_and_update_config) setattr( NPUPlatform, _ASCEND_PLATFORM_PATCH_ATTR, - original_fix_incompatible_config, + original_check_and_update_config, ) + return True -def _snapshot_afd_dbo_config(vllm_config: VllmConfig) -> dict[str, bool | int] | None: +def _snapshot_afd_dbo_config( + vllm_config: VllmConfig, +) -> _AFDDBOConfigSnapshot | None: if not _has_valid_afd_config(vllm_config): return None parallel_config = vllm_config.parallel_config - return { - "enable_dbo": parallel_config.enable_dbo, - "use_ubatching": parallel_config.use_ubatching, - "ubatch_size": parallel_config.ubatch_size, - } + return _AFDDBOConfigSnapshot( + enable_dbo=parallel_config.enable_dbo, + ubatch_size=parallel_config.ubatch_size, + all2all_backend=parallel_config.all2all_backend, + ) def _restore_afd_dbo_config( vllm_config: VllmConfig, - saved: dict[str, bool | int], + saved: _AFDDBOConfigSnapshot, ) -> None: parallel_config = vllm_config.parallel_config - if not ( - saved["enable_dbo"] - or saved["use_ubatching"] - or int(saved["ubatch_size"] or 0) != 0 - ): + if not saved.enable_dbo and saved.ubatch_size == 0: return - parallel_config.enable_dbo = saved["enable_dbo"] - parallel_config.ubatch_size = saved["ubatch_size"] + parallel_config.enable_dbo = saved.enable_dbo + parallel_config.ubatch_size = saved.ubatch_size + parallel_config.all2all_backend = saved.all2all_backend def _has_valid_afd_config(vllm_config: VllmConfig) -> bool: try: return parse_optional_afd_config(vllm_config, validate=True) is not None - except Exception: + except (TypeError, ValueError): return False diff --git a/afd_plugin/v1/worker/npu/attention_model_runner.py b/afd_plugin/v1/worker/npu/attention_model_runner.py index f297d981..3fd69e9c 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner.py @@ -5,12 +5,14 @@ from __future__ import annotations import copy +from contextlib import AbstractContextManager, nullcontext from functools import partial from typing import Any import numpy as np import torch import torch.distributed as dist +import torch.nn as nn from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size @@ -25,12 +27,20 @@ from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder -from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec -from vllm_ascend.ascend_forward_context import ( - select_moe_comm_method, - set_ascend_forward_context, -) +from vllm.v1.attention.backends.utils import CommonAttentionMetadata +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, KVCacheConfig +from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.worker.ubatch_utils import UBatchSlices +from vllm_ascend.ascend_forward_context import set_ascend_forward_context from vllm_ascend.attention.attention_v1 import AscendAttentionState +from vllm_ascend.attention.context_parallel.dsa_cp import ( + AscendDSACPMetadataBuilder, +) +from vllm_ascend.attention.context_parallel.sfa_cp import ( + AscendSFADCPMetadataBuilder, +) +from vllm_ascend.attention.dsa_v1 import AscendDSAMetadataBuilder from vllm_ascend.attention.utils import ( AscendCommonAttentionMetadata, using_paged_attention, @@ -39,13 +49,21 @@ from vllm_ascend.ops.rotary_embedding import update_cos_sin from vllm_ascend.spec_decode.dflash_proposer import AscendDflashProposer from vllm_ascend.spec_decode.draft_proposer import AscendDraftModelProposer +from vllm_ascend.spec_decode.dspark_proposer import AscendDSparkProposer from vllm_ascend.spec_decode.eagle_proposer import AscendEagleProposer +from vllm_ascend.spec_decode.step3p5 import AscendStep3p5MTPProposer from vllm_ascend.utils import ( + embedding_tp_enable, enable_sp, lmhead_tp_enable, + oproj_tp_enable, should_skip_allreduce_across_dp_group, ) -from vllm_ascend.worker.model_runner_v1 import NPUModelRunner +from vllm_ascend.worker.model_runner_v1 import ( + SEQ_LEN_WITH_MAX_PA_WORKSPACE, + NPUModelRunner, + PerLayerAttnMetadata, +) from afd_plugin.compat.npu import ( fail_if_unsupported_npu_afd_features, @@ -92,7 +110,7 @@ class AFDNPUAttentionModelRunner(NPUModelRunner): afd_expected_role = "attention" - def __init__(self, vllm_config: VllmConfig, device: object) -> None: + def __init__(self, vllm_config: VllmConfig, device: torch.device): afd_config = self.parse_config(vllm_config) super().__init__(vllm_config, device) @@ -125,6 +143,7 @@ def __init__(self, vllm_config: VllmConfig, device: object) -> None: self._afd_suppress_metadata_send = False self._afd_transaction_counter = 0 self._afd_async_moe_ubatch_metadata = None + self._afd_live_execution = False self.ubatch_slices = None self.prof = create_afd_npu_profiler("attention") @@ -132,29 +151,50 @@ def __init__(self, vllm_config: VllmConfig, device: object) -> None: def parse_config(vllm_config: VllmConfig) -> AFDConfig: return parse_afd_config(vllm_config, expected_role="attention") - def execute_model(self, *args: Any, **kwargs: Any) -> Any: + # Patch reason: vLLM-Ascend calls the execution/padding hook without opting + # into microbatching, and AFD must keep that hook's upstream default intact. + # Patch functionality: scope an AFD live-execution flag around the delegated + # upstream request so the hook can distinguish live requests from dummy runs. + # Signature: matches upstream; no added parameters. + def execute_model( + self, + scheduler_output: SchedulerOutput, + intermediate_tensors: IntermediateTensors | None = None, + ) -> ModelRunnerOutput | IntermediateTensors | None: step_afd_npu_profiler(self.prof) - return super().execute_model(*args, **kwargs) + # ### PATCH START: AFD live execution scope + self._afd_live_execution = True + try: + result = super().execute_model(scheduler_output, intermediate_tensors) + finally: + self._afd_live_execution = False + # ### PATCH END: AFD live execution scope + return result - def _model_forward(self, *args: Any, **kwargs: Any) -> Any: + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner._model_forward. + # Patch reason: the upstream forward path does not install AFD stage metadata + # or expose Ascend ubatch slices to the model wrapper. + # Patch functionality: inject AFD forward-context state while retaining the + # upstream model invocation, ENPU ordering, and FlashComm output handling. + # Signature: matches upstream; no added parameters. + def _model_forward( + self, + num_tokens_padded: int, + input_ids: torch.Tensor | None = None, + positions: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **model_kwargs: dict[str, Any], + ): forward_context = get_forward_context() + # ### PATCH START: AFD forward-context metadata if self.ubatch_slices is not None: forward_context.ubatch_slices = self.ubatch_slices - try: - forward_context.dbo_enabled = bool(forward_context.dbo_enabled) - except AttributeError: - forward_context.dbo_enabled = False + forward_context.dbo_enabled = False self._install_afd_metadata_on_forward_context(forward_context) self._install_async_moe_ubatch_metadata_on_forward_context(forward_context) - - ( - num_tokens_padded, - input_ids, - positions, - intermediate_tensors, - inputs_embeds, - model_kwargs, - ) = _model_forward_values(args, kwargs) + # ### PATCH END: AFD forward-context metadata assert self.model is not None model_inputs: dict[str, Any] = { @@ -179,56 +219,132 @@ def _model_forward(self, *args: Any, **kwargs: Any) -> Any: num_tokens_padded, ) + # ### PATCH START: AFD defers FlashComm gather to the ubatch wrapper if ( forward_context.flash_comm_v1_enabled and not forward_context.dbo_enabled and not isinstance(hidden_states, IntermediateTensors) ): hidden_states = self._all_gather_hidden_states_and_aux(hidden_states) + # ### PATCH END: AFD defers FlashComm gather to the ubatch wrapper return hidden_states - def _build_attention_metadata(self, *args: Any, **kwargs: Any) -> Any: - values = _attention_metadata_values(args, kwargs) + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner._build_attention_metadata. + # Patch reason: upstream accepts ubatch slices but does not construct separate + # Ascend attention metadata for each NPU ubatch. + # Patch functionality: normalize padded slices, build AFD control metadata, + # and route only split batches through the plugin-owned metadata builder. + # Signature: matches upstream; no added parameters. + def _build_attention_metadata( + self, + num_tokens: int, + num_reqs: int, + max_query_len: int, + num_tokens_padded: int | None = None, + num_reqs_padded: int | None = None, + ubatch_slices: UBatchSlices | None = None, + logits_indices: torch.Tensor | None = None, + use_spec_decode: bool = False, + for_cudagraph_capture: bool = False, + num_scheduled_tokens: dict[str, int] | None = None, + num_scheduled_tokens_np: np.ndarray | None = None, + cascade_attn_prefix_lens: list[list[int]] | None = None, + ) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]: + # ### PATCH START: AFD NPU ubatch metadata routing ubatch_slices = _normalize_metadata_ubatch_slices( - values.get("ubatch_slices"), - values, + ubatch_slices, + num_tokens_padded, + num_reqs_padded, ) - if ubatch_slices is not values.get("ubatch_slices"): - args, kwargs = _replace_attention_metadata_ubatch_slices( - args, - kwargs, - ubatch_slices, - ) if self.afd_async_extra_info.async_moe_ubatching: self.ubatch_slices = None return self._build_attention_metadata_with_async_moe_ubatches( - args, - kwargs, - values, + num_tokens=num_tokens, + num_reqs=num_reqs, + max_query_len=max_query_len, + num_tokens_padded=num_tokens_padded, + num_reqs_padded=num_reqs_padded, + ubatch_slices=ubatch_slices, + logits_indices=logits_indices, + use_spec_decode=use_spec_decode, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens_np=num_scheduled_tokens_np, + cascade_attn_prefix_lens=cascade_attn_prefix_lens, ) self._afd_pending_metadata = self._build_afd_metadata( ubatch_slices, - int(values.get("num_tokens", 0)), + num_tokens, ) self.ubatch_slices = ubatch_slices if ubatch_slices is not None: - return self._build_attention_metadata_with_ubatches(*args, **kwargs) - return super()._build_attention_metadata(*args, **kwargs) + return self._build_attention_metadata_with_ubatches( + num_tokens=num_tokens, + num_reqs=num_reqs, + max_query_len=max_query_len, + num_tokens_padded=num_tokens_padded, + num_reqs_padded=num_reqs_padded, + ubatch_slices=ubatch_slices, + logits_indices=logits_indices, + use_spec_decode=use_spec_decode, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens_np=num_scheduled_tokens_np, + cascade_attn_prefix_lens=cascade_attn_prefix_lens, + ) + result = super()._build_attention_metadata( + num_tokens=num_tokens, + num_reqs=num_reqs, + max_query_len=max_query_len, + num_tokens_padded=num_tokens_padded, + num_reqs_padded=num_reqs_padded, + ubatch_slices=ubatch_slices, + logits_indices=logits_indices, + use_spec_decode=use_spec_decode, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens_np=num_scheduled_tokens_np, + cascade_attn_prefix_lens=cascade_attn_prefix_lens, + ) + # ### PATCH END: AFD NPU ubatch metadata routing + return result def _build_attention_metadata_with_async_moe_ubatches( self, - args: tuple[Any, ...], - kwargs: dict[str, Any], - values: dict[str, Any], - ) -> Any: - full_metadata = super()._build_attention_metadata(*args, **kwargs) + num_tokens: int, + num_reqs: int, + max_query_len: int, + num_tokens_padded: int | None, + num_reqs_padded: int | None, + ubatch_slices: UBatchSlices | None, + logits_indices: torch.Tensor | None, + use_spec_decode: bool, + for_cudagraph_capture: bool, + num_scheduled_tokens: dict[str, int] | None, + num_scheduled_tokens_np: np.ndarray | None, + cascade_attn_prefix_lens: list[list[int]] | None, + ) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]: + full_metadata = super()._build_attention_metadata( + num_tokens=num_tokens, + num_reqs=num_reqs, + max_query_len=max_query_len, + num_tokens_padded=num_tokens_padded, + num_reqs_padded=num_reqs_padded, + ubatch_slices=ubatch_slices, + logits_indices=logits_indices, + use_spec_decode=use_spec_decode, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens_np=num_scheduled_tokens_np, + cascade_attn_prefix_lens=cascade_attn_prefix_lens, + ) self._afd_async_moe_ubatch_metadata = None self._afd_pending_metadata = self._build_afd_metadata( None, - int(values.get("num_tokens", 0)), + num_tokens, ) - num_scheduled_tokens_np = values.get("num_scheduled_tokens_np") if num_scheduled_tokens_np is None: return full_metadata @@ -244,7 +360,7 @@ def _build_attention_metadata_with_async_moe_ubatches( "num_scheduled_tokens=%s request_slices=%s token_slices=%s " "stage_num_tokens=%s", len(num_scheduled_tokens_np), - int(values.get("num_tokens", 0)), + num_tokens, num_scheduled_tokens_np.tolist(), [ (ubatch_slice.request_slice.start, ubatch_slice.request_slice.stop) @@ -257,18 +373,23 @@ def _build_attention_metadata_with_async_moe_ubatches( [int(ubatch_slice.num_tokens) for ubatch_slice in ubatch_slices], ) - stage_args, stage_kwargs = _replace_attention_metadata_ubatch_slices( - args, - kwargs, - ubatch_slices, - ) stage_attn_metadata, _ = self._build_attention_metadata_with_ubatches( - *stage_args, - **stage_kwargs, + num_tokens=num_tokens, + num_reqs=num_reqs, + max_query_len=max_query_len, + num_tokens_padded=num_tokens_padded, + num_reqs_padded=num_reqs_padded, + ubatch_slices=ubatch_slices, + logits_indices=logits_indices, + use_spec_decode=use_spec_decode, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens_np=num_scheduled_tokens_np, + cascade_attn_prefix_lens=cascade_attn_prefix_lens, ) self._afd_pending_metadata = self._build_afd_metadata( ubatch_slices, - int(values.get("num_tokens", 0)), + num_tokens, ) self._afd_async_moe_ubatch_metadata = { "attn_metadata": stage_attn_metadata, @@ -276,6 +397,13 @@ def _build_attention_metadata_with_async_moe_ubatches( } return full_metadata + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner._build_attention_metadata. + # Patch reason: upstream builds one metadata object even when AFD schedules + # two NPU execution stages. + # Patch functionality: copy the pinned upstream builders and emit one + # PerLayerAttnMetadata mapping per AFD ubatch. + # Signature: matches the upstream metadata hook; no added parameters. def _build_attention_metadata_with_ubatches( self, num_tokens: int, @@ -283,14 +411,14 @@ def _build_attention_metadata_with_ubatches( max_query_len: int, num_tokens_padded: int | None = None, num_reqs_padded: int | None = None, - ubatch_slices: Any | None = None, - logits_indices: Any | None = None, + ubatch_slices: UBatchSlices | None = None, + logits_indices: torch.Tensor | None = None, use_spec_decode: bool = False, for_cudagraph_capture: bool = False, num_scheduled_tokens: dict[str, int] | None = None, num_scheduled_tokens_np: np.ndarray | None = None, cascade_attn_prefix_lens: list[list[int]] | None = None, - ) -> tuple[Any, Any | None]: + ) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]: """Build per-ubatch Ascend attention metadata. Builds the DBO-specific metadata layout required by Ascend ubatching @@ -299,14 +427,14 @@ def _build_attention_metadata_with_ubatches( if len(self.kv_cache_config.kv_cache_groups) == 0: return {}, None + # ### PATCH START: AFD per-ubatch metadata containers assert ubatch_slices is not None + attn_metadata: list[dict[str, Any]] = [ + dict() for _ in range(len(ubatch_slices)) + ] + # ### PATCH END: AFD per-ubatch metadata containers num_tokens_padded = num_tokens_padded or num_tokens num_reqs_padded = num_reqs_padded or num_reqs - attn_metadata: Any = [dict() for _ in range(len(ubatch_slices))] - - if self._seq_lens_cpu_event_pending and self._seq_lens_cpu_event is not None: - self._seq_lens_cpu_event.synchronize() - self._seq_lens_cpu_event_pending = False if for_cudagraph_capture: max_seq_len = self.max_model_len @@ -315,6 +443,28 @@ def _build_attention_metadata_with_ubatches( kv_cache_groups = self.kv_cache_config.kv_cache_groups + def _get_dcp_metadata(block_table_tensor: torch.Tensor): + if not self.use_dcp: + return None, block_table_tensor + + fixed_decode_seq_lens_cpu = None + if self.use_async_spec_decode: + fixed_decode_seq_lens_cpu = self.optimistic_seq_lens_cpu[ + :num_reqs + ].numpy() + + assert num_reqs_padded is not None + return self.dcp_manager.generate_dcp_metadata( + num_tokens, + self.query_lens, + self.input_batch, + num_scheduled_tokens_np, + block_table_tensor, + num_reqs_padded, + num_reqs, + fixed_decode_seq_lens_cpu, + ) + def _get_block_table_and_slot_mapping(kv_cache_gid: int): assert num_reqs_padded is not None and num_tokens_padded is not None kv_cache_spec = kv_cache_groups[kv_cache_gid].kv_cache_spec @@ -335,14 +485,29 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): blk_table_tensor = blk_table.get_device_tensor()[:num_reqs_padded] slot_mapping[num_tokens:num_tokens_padded].fill_(-1) blk_table_tensor[num_reqs:num_reqs_padded].fill_(0) - if self.model_config.enable_return_routed_experts and kv_cache_gid == 0: - self.cpu_slot_mapping = slot_mapping.cpu().numpy() + if ( + self.model_config.enable_return_routed_experts + and kv_cache_gid == 0 + and self.routed_experts_initialized + ): + num_slots = slot_mapping.shape[0] + self.routed_experts_slot_mapping_device[:num_slots].copy_( + slot_mapping, + ) return blk_table_tensor, slot_mapping block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0) + self.long_seq_metadata, block_table_gid_0 = _get_dcp_metadata( + block_table_gid_0, + ) num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[ :num_reqs_padded ] + num_prompt_tokens_cpu = self.input_batch.num_prompt_tokens_cpu_tensor[ + :num_reqs_padded + ] + is_prefilling = num_computed_tokens_cpu < num_prompt_tokens_cpu + is_prefilling[num_reqs:] = False seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded] if self.use_async_spec_decode: seq_lens_cpu = None @@ -353,6 +518,7 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1], seq_lens=self.seq_lens[:num_reqs_padded], _seq_lens_cpu=self.optimistic_seq_lens_cpu[:num_reqs_padded], + seq_lens_cpu_upper_bound=self.optimistic_seq_lens_cpu[:num_reqs_padded], seq_lens_cpu=seq_lens_cpu, num_computed_tokens_cpu=num_computed_tokens_cpu, num_reqs=num_reqs_padded, @@ -362,11 +528,17 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): block_table_tensor=block_table_gid_0, slot_mapping=slot_mapping_gid_0, causal=True, + is_prefilling=is_prefilling, num_input_tokens=num_tokens_padded, actual_seq_lengths_q=self.actual_seq_lengths_q, positions=self.positions, + positions_cpu=self._dsa_positions_cpu_buf if self.use_compress else None, attn_state=self.attn_state, decode_token_per_req=self.decode_token_per_req, + context_parallel_metadata=self.long_seq_metadata, + group_len=self.group_len.gpu[:num_reqs_padded], + group_key_idx=self.group_key_idx.gpu[:num_reqs_padded], + group_key_cache_idx=self.group_key_cache_idx.gpu[:num_reqs_padded], ) if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill: @@ -378,7 +550,10 @@ def _get_block_table_and_slot_mapping(kv_cache_gid: int): def _build_attn_group_metadata( kv_cache_gid: int, attn_gid: int, - common_attn_metadata: AscendCommonAttentionMetadata, + common_attn_metadata: CommonAttentionMetadata, + prefill_ratio_to_sas_metadata: dict[Any, Any], + decode_ratio_to_sas_metadata: dict[Any, Any], + common_ratio_to_sas_metadata: dict[Any, Any], ubid: int | None = None, ) -> None: attn_group = self.attn_groups[kv_cache_gid][attn_gid] @@ -399,7 +574,28 @@ def _build_attn_group_metadata( ], ) - if for_cudagraph_capture: + if isinstance( + builder, + AscendDSAMetadataBuilder | AscendDSACPMetadataBuilder, + ): + if for_cudagraph_capture: + prefill_ratio_to_sas_metadata = {} + decode_ratio_to_sas_metadata = {} + common_ratio_to_sas_metadata = {} + extra_attn_metadata_args = dict( + num_reqs_actual=num_reqs, + prefill_ratio_to_sas_metadata=prefill_ratio_to_sas_metadata, + decode_ratio_to_sas_metadata=decode_ratio_to_sas_metadata, + common_ratio_to_sas_metadata=common_ratio_to_sas_metadata, + block_size=attn_group.kv_cache_spec.block_size, + ) + + if for_cudagraph_capture and not isinstance( + builder, + AscendDSAMetadataBuilder + | AscendDSACPMetadataBuilder + | AscendSFADCPMetadataBuilder, + ): attn_metadata_i = builder.build_for_cudagraph_capture( common_attn_metadata, ) @@ -420,12 +616,21 @@ def _build_attn_group_metadata( attn_metadata_i.spec_state_indices_tensor[ attn_metadata_i.num_spec_decodes : ].fill_(0) + if isinstance(builder, AscendDSAMetadataBuilder): + prefill_ratio_to_sas_metadata = builder.prefill_ratio_to_sas_metadata + decode_ratio_to_sas_metadata = builder.decode_ratio_to_sas_metadata + common_ratio_to_sas_metadata = builder.common_ratio_to_sas_metadata + # ### PATCH START: AFD per-ubatch metadata assignment assert ubid is not None attn_metadata_dict = attn_metadata[ubid] for layer_name in attn_group.layer_names: attn_metadata_dict[layer_name] = attn_metadata_i + # ### PATCH END: AFD per-ubatch metadata assignment + prefill_ratio_to_sas_metadata: dict[Any, Any] = {} + decode_ratio_to_sas_metadata: dict[Any, Any] = {} + common_ratio_to_sas_metadata: dict[Any, Any] = {} spec_decode_common_attn_metadata = None for kv_cache_gid, kv_cache_group in enumerate( self.kv_cache_config.kv_cache_groups, @@ -452,25 +657,45 @@ def _build_attn_group_metadata( kv_cache_gid, ) ) + if self.speculative_config and isinstance( + self.drafter, + AscendStep3p5MTPProposer | AscendDSparkProposer, + ): + self.drafter.set_per_group_attn_metadata( + kv_cache_gid, + cm.block_table_tensor, + cm.slot_mapping, + ) if self.speculative_config and spec_decode_common_attn_metadata is None: if isinstance( self.drafter, AscendEagleProposer | AscendDraftModelProposer - | AscendDflashProposer, + | AscendDflashProposer + | AscendDSparkProposer, ): if self.drafter.attn_layer_names[0] in kv_cache_group.layer_names: spec_decode_common_attn_metadata = cm else: spec_decode_common_attn_metadata = cm for attn_gid in range(len(self.attn_groups[kv_cache_gid])): + # ### PATCH START: AFD common-metadata split ubatch_common_metadata = split_attn_metadata( ubatch_slices, cm, num_tokens_padded, ) for ubid, ubatch_cm in enumerate(ubatch_common_metadata): - _build_attn_group_metadata(kv_cache_gid, attn_gid, ubatch_cm, ubid) + _build_attn_group_metadata( + kv_cache_gid, + attn_gid, + ubatch_cm, + prefill_ratio_to_sas_metadata, + decode_ratio_to_sas_metadata, + common_ratio_to_sas_metadata, + ubid, + ) + # ### PATCH END: AFD common-metadata split if self.is_mm_prefix_lm: req_doc_ranges = {} @@ -483,9 +708,11 @@ def _build_attn_group_metadata( image_doc_ranges.extend(img_doc_range) req_idx = self.input_batch.req_id_to_index[req_id] req_doc_ranges[req_idx] = image_doc_ranges + # ### PATCH START: AFD multimodal metadata assignment for ub_metadata in attn_metadata: for metadata in ub_metadata.values(): metadata.mm_prefix_range = req_doc_ranges + # ### PATCH END: AFD multimodal metadata assignment if spec_decode_common_attn_metadata is not None and ( num_reqs != num_reqs_padded or num_tokens != num_tokens_padded @@ -502,7 +729,7 @@ def _dummy_run( self, num_tokens: int, with_prefill: bool = False, - cudagraph_runtime_mode: Any | None = None, + cudagraph_runtime_mode: CUDAGraphMode | None = None, force_attention: bool = False, uniform_decode: bool = False, is_profile: bool = False, @@ -514,11 +741,7 @@ def _dummy_run( num_active_loras: int = 0, profile_seq_lens: int | None = None, profile_cpp: bool = False, - count_prof_step: bool = False, - ) -> Any: - if count_prof_step: - step_afd_npu_profiler(self.prof) - + ) -> tuple[torch.Tensor, torch.Tensor]: with torch.inference_mode(): return self._dummy_run_inference_mode( num_tokens, @@ -541,7 +764,7 @@ def _dummy_run_inference_mode( self, num_tokens: int, with_prefill: bool = False, - cudagraph_runtime_mode: Any | None = None, + cudagraph_runtime_mode: CUDAGraphMode | None = None, force_attention: bool = False, uniform_decode: bool = False, is_profile: bool = False, @@ -553,7 +776,7 @@ def _dummy_run_inference_mode( num_active_loras: int = 0, profile_seq_lens: int | None = None, profile_cpp: bool = False, - ) -> Any: + ) -> tuple[torch.Tensor, torch.Tensor]: previous = self._afd_is_graph_capturing self._afd_is_graph_capturing = bool(is_graph_capturing) if not ( @@ -605,7 +828,22 @@ def _dummy_run_inference_mode( self._afd_pending_metadata = None self._afd_async_moe_ubatch_metadata = None - def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: + # Upstream source: vLLM commit 68b0c3135, + # GPUModelRunner._warmup_and_capture. + # Patch reason: AFD needs both single-stage and two-stage Ascend graph keys, + # because live decode may fall below the DBO threshold. + # Patch functionality: run the pinned warmup/capture hook once for each AFD + # execution shape while coordinating metadata with the FFN workers. + # Signature: matches upstream; no added parameters. + def _warmup_and_capture( + self, + desc: BatchDescriptor, + cudagraph_runtime_mode: CUDAGraphMode, + profile_seq_lens: int | None = None, + allow_microbatching: bool = False, + num_warmups: int | None = None, + profiler: AbstractContextManager[Any] | None = None, + ): """Capture both single-stage and ubatched FFN graph keys. Native vLLM only captures the ubatched graph when microbatching is @@ -614,25 +852,11 @@ def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: still produce a single-stage key below the ubatch threshold. """ - names = [ - "desc", - "cudagraph_runtime_mode", - "profile_seq_lens", - "allow_microbatching", - "num_warmups", - ] - values = dict(zip(names, args, strict=False)) - values.update(kwargs) - desc = values.get("desc") - cudagraph_runtime_mode = values.get("cudagraph_runtime_mode") - if desc is None or cudagraph_runtime_mode is None: - return super()._warmup_and_capture(*args, **kwargs) - - num_warmups = values.get("num_warmups") + # ### PATCH START: AFD dual graph capture + if profiler is None: + profiler = nullcontext() if num_warmups is None: num_warmups = self.compilation_config.cudagraph_num_of_warmups - allow_microbatching = bool(values.get("allow_microbatching", False)) - profile_seq_lens = values.get("profile_seq_lens") if allow_microbatching: self._afd_warmup_and_capture_once( @@ -641,29 +865,30 @@ def _warmup_and_capture(self, *args: Any, **kwargs: Any) -> Any: profile_seq_lens=profile_seq_lens, allow_microbatching=False, num_warmups=int(num_warmups), - cudagraph_mode_cls=CUDAGraphMode, + profiler=nullcontext(), ) - return self._afd_warmup_and_capture_once( + self._afd_warmup_and_capture_once( desc=desc, cudagraph_runtime_mode=cudagraph_runtime_mode, profile_seq_lens=profile_seq_lens, allow_microbatching=allow_microbatching, num_warmups=int(num_warmups), - cudagraph_mode_cls=CUDAGraphMode, + profiler=profiler, ) + # ### PATCH END: AFD dual graph capture def _afd_warmup_and_capture_once( self, *, - desc: Any, - cudagraph_runtime_mode: Any, + desc: BatchDescriptor, + cudagraph_runtime_mode: CUDAGraphMode, profile_seq_lens: int | None, allow_microbatching: bool, num_warmups: int, - cudagraph_mode_cls: Any, - ) -> Any: - force_attention = cudagraph_runtime_mode == cudagraph_mode_cls.FULL + profiler: AbstractContextManager[Any], + ) -> None: + force_attention = cudagraph_runtime_mode == CUDAGraphMode.FULL previous_is_warmup = bool(self._is_warmup) try: @@ -671,7 +896,7 @@ def _afd_warmup_and_capture_once( for _ in range(num_warmups): self._dummy_run( desc.num_tokens, - cudagraph_runtime_mode=cudagraph_mode_cls.NONE, + cudagraph_runtime_mode=CUDAGraphMode.NONE, force_attention=force_attention, uniform_decode=desc.uniform, allow_microbatching=allow_microbatching, @@ -702,27 +927,40 @@ def _afd_warmup_and_capture_once( ) self._afd_suppress_metadata_send = True - return self._dummy_run( - desc.num_tokens, - cudagraph_runtime_mode=cudagraph_runtime_mode, - uniform_decode=desc.uniform, - allow_microbatching=allow_microbatching, - skip_eplb=True, - remove_lora=False, - num_active_loras=desc.num_active_loras, - is_graph_capturing=True, - profile_seq_lens=profile_seq_lens, - ) + with ( + profiler, + torch.profiler.record_function( + f"capture_{desc.num_tokens}_{cudagraph_runtime_mode.name}", + ), + ): + self._dummy_run( + desc.num_tokens, + cudagraph_runtime_mode=cudagraph_runtime_mode, + uniform_decode=desc.uniform, + allow_microbatching=allow_microbatching, + skip_eplb=True, + remove_lora=False, + num_active_loras=desc.num_active_loras, + is_graph_capturing=True, + profile_seq_lens=profile_seq_lens, + ) finally: self._afd_is_graph_capturing = previous_is_graph_capturing self._afd_suppress_metadata_send = previous_suppress_send self._afd_pending_metadata = previous_metadata + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner._dummy_run. + # Patch reason: upstream's dummy path forces ubatch slices to None, so it + # cannot warm or capture the AFD two-stage Ascend execution path. + # Patch functionality: preserve the pinned upstream dummy setup while + # constructing and forwarding the same two ubatches used by live requests. + # Signature: matches upstream; no added parameters. def _dummy_run_with_ubatches( self, num_tokens: int, with_prefill: bool = False, - cudagraph_runtime_mode: Any | None = None, + cudagraph_runtime_mode: CUDAGraphMode | None = None, force_attention: bool = False, uniform_decode: bool = False, is_profile: bool = False, @@ -734,7 +972,7 @@ def _dummy_run_with_ubatches( num_active_loras: int = 0, profile_seq_lens: int | None = None, profile_cpp: bool = False, - ) -> Any: + ) -> tuple[torch.Tensor, torch.Tensor]: assert ( cudagraph_runtime_mode is None or cudagraph_runtime_mode.valid_runtime_modes() @@ -770,6 +1008,7 @@ def _dummy_run_with_ubatches( self.query_lens = torch.from_numpy(num_scheduled_tokens) num_tokens_unpadded = int(num_scheduled_tokens.sum()) num_sampled_tokens = np.ones(num_reqs, dtype=np.int32) + # ### PATCH START: AFD dummy ubatch decision ( _cudagraph_mode, batch_desc, @@ -790,6 +1029,19 @@ def _dummy_run_with_ubatches( force_has_lora=num_active_loras > 0, force_num_active_loras=num_active_loras, ) + # ### PATCH END: AFD dummy ubatch decision + if self.use_dcp: + self.dcp_manager.init_batch_info( + num_scheduled_tokens, + num_reqs, + self.input_batch.num_computed_tokens_cpu, + self.input_batch.num_prompt_tokens, + ) + if self.speculative_config: + self.dcp_manager.query_lens_full.cpu[:num_reqs] = torch.from_numpy( + num_scheduled_tokens, + ) + self.dcp_manager.query_lens_full.copy_to_gpu() if cudagraph_runtime_mode is None: cudagraph_runtime_mode = _cudagraph_mode else: @@ -806,82 +1058,109 @@ def _dummy_run_with_ubatches( num_tokens_across_dp[:] = num_tokens_padded num_scheduled_tokens = num_scheduled_tokens.repeat(num_reqs_padded) + if self.dynamic_eplb: + self.update_eplb_heat_collection_status(num_tokens_padded) + ubatch_slices, ubatch_slices_padded = None, None - attn_metadata = None - if self._should_build_dummy_attn_metadata( - force_attention, - is_profile, - cudagraph_runtime_mode, - ): - self.attn_state = AscendAttentionState.DecodeOnly - if self.speculative_config and self.speculative_config.method == "mtp": - if self.vllm_config.model_config.use_mla: - self.attn_state = AscendAttentionState.SpecDecoding + attn_metadata: PerLayerAttnMetadata | None = None + with self.synchronize_input_prep(): + if self._should_build_dummy_attn_metadata( + force_attention, + is_profile, + cudagraph_runtime_mode, + ): + self.attn_state = AscendAttentionState.DecodeOnly + if self.speculative_config and self.speculative_config.method == "mtp": + if self.vllm_config.model_config.use_mla: + self.attn_state = AscendAttentionState.SpecDecoding + else: + self.attn_state = AscendAttentionState.ChunkedPrefill + if profile_seq_lens is not None: + seq_lens = profile_seq_lens else: - self.attn_state = AscendAttentionState.ChunkedPrefill - if profile_seq_lens is not None: - seq_lens = profile_seq_lens - else: - seq_lens = ( - 6144 - if is_graph_capturing - and using_paged_attention(num_tokens, self.vllm_config) - else max_query_len - ) + seq_lens = ( + SEQ_LEN_WITH_MAX_PA_WORKSPACE + if is_graph_capturing + and using_paged_attention(num_tokens, self.vllm_config) + else max_query_len + ) - self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens - self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) - self.seq_lens.copy_(self.optimistic_seq_lens_cpu, non_blocking=True) + self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens + self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) + self.seq_lens.copy_( + self.optimistic_seq_lens_cpu, + non_blocking=True, + ) - cum_num_tokens = self._get_cumsum_and_arange( - num_scheduled_tokens, - self.query_pos.np, - ) - self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.query_start_loc.copy_to_gpu() - if self._has_gdn: - self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.gdn_query_start_loc.copy_to_gpu() + cum_num_tokens = self._get_cumsum_and_arange( + num_scheduled_tokens, + self.query_pos.np, + ) + self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens + self.query_start_loc.copy_to_gpu() + if self._has_gdn: + self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = ( + cum_num_tokens + ) + self.gdn_query_start_loc.copy_to_gpu() + + if not profile_cpp: + num_reqs_padded = self._pad_query_start_loc_for_fia( + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_runtime_mode, + batch_desc.num_reqs, + ) - if not profile_cpp: - num_reqs_padded = self._pad_query_start_loc_for_fia( + self.input_batch.block_table.commit_block_table(num_reqs_padded) + pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL + # ### PATCH START: AFD dummy ubatch slices + ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices( + should_ubatch, + num_scheduled_tokens, num_tokens_padded, num_reqs_padded, - num_reqs, - cudagraph_runtime_mode, - batch_desc.num_reqs, + self.vllm_config, ) - - pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL - ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices( - should_ubatch, - num_scheduled_tokens, - num_tokens_padded, - num_reqs_padded, - self.vllm_config, - ) - self.ubatch_slices = ubatch_slices_padded if pad_attn else ubatch_slices - attn_metadata, _ = self._build_attention_metadata( - num_tokens=num_tokens_unpadded, - num_tokens_padded=num_tokens_padded, - num_reqs=num_reqs_padded, - max_query_len=max_query_len, - ubatch_slices=self.ubatch_slices, - for_cudagraph_capture=is_graph_capturing, - num_scheduled_tokens_np=num_scheduled_tokens, - ) - elif should_ubatch: - pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL - ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices( - should_ubatch, - num_scheduled_tokens, - num_tokens_padded, - num_reqs_padded, - self.vllm_config, - ) - self.ubatch_slices = ubatch_slices_padded if pad_attn else ubatch_slices - else: - self.ubatch_slices = None + self.ubatch_slices = ubatch_slices_padded if pad_attn else ubatch_slices + # ### PATCH END: AFD dummy ubatch slices + if self.use_compress: + self.positions.fill_(127) + self._dsa_positions_cpu_buf.fill_(127) + attn_metadata, _ = self._build_attention_metadata( + num_tokens=num_tokens_unpadded, + num_tokens_padded=num_tokens_padded, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + # ### PATCH START: AFD dummy ubatch metadata input + ubatch_slices=self.ubatch_slices, + # ### PATCH END: AFD dummy ubatch metadata input + for_cudagraph_capture=is_graph_capturing, + num_scheduled_tokens_np=num_scheduled_tokens, + ) + if not is_graph_capturing: + for kv_cache_gid in range( + len(self.kv_cache_config.kv_cache_groups), + ): + block_table = self.input_batch.block_table[kv_cache_gid] + block_table.slot_mapping.gpu.fill_(-1) + # ### PATCH START: AFD attention-free dummy ubatch slices + elif should_ubatch: + pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL + ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices( + should_ubatch, + num_scheduled_tokens, + num_tokens_padded, + num_reqs_padded, + self.vllm_config, + ) + self.ubatch_slices = ubatch_slices_padded if pad_attn else ubatch_slices + else: + self.ubatch_slices = None + # ### PATCH END: AFD attention-free dummy ubatch slices with self.maybe_dummy_run_with_lora( self.lora_config, @@ -969,6 +1248,11 @@ def dummy_drafter_compute_logits(hidden_states): aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_desc, model_instance=self.model, + has_sinks=self._has_sinks, + input_ids=input_ids, + eplb_heat_collection_status=( + self.eplb_heat_collection_status if self.dynamic_eplb else False + ), ): outputs = self._model_forward( num_tokens_padded, @@ -983,7 +1267,7 @@ def dummy_drafter_compute_logits(hidden_states): hidden_states = outputs dummy_compute_logits(hidden_states) - if self.drafter: + if self.drafter and not profile_cpp: self.drafter.dummy_run( num_tokens=num_tokens_padded, with_prefill=with_prefill, @@ -996,19 +1280,18 @@ def dummy_drafter_compute_logits(hidden_states): is_profile=is_profile, ) if is_profile and self.dynamic_eplb: - target = ( - self.model.language_model - if hasattr(self.model, "language_model") - else self.model - ) - target.clear_all_moe_loads() - if self.dynamic_eplb: - self.eplb_updator.forward_end() + self.eplb_updator.adaptor.clear_all_moe_loads() + if not is_profile and self.dynamic_eplb: + self.eplb_updator.forward_end(self.eplb_heat_collection_status) + self._finalize_dump_data(dump=False) + if self.use_compress and force_attention: + self.positions.fill_(0) + self._dsa_positions_cpu_buf.fill_(0) return hidden_states, hidden_states def _build_afd_metadata( self, - ubatch_slices: Any, + ubatch_slices: UBatchSlices | None, num_tokens_unpadded: int, ) -> AFDForwardContextMetadata: if ubatch_slices and len(ubatch_slices) > 1: @@ -1050,7 +1333,7 @@ def _install_afd_metadata_on_forward_context( forward_context.additional_kwargs["afd_metadata"] = self._afd_pending_metadata if self.connector.control_plane is None: return - if bool(getattr(self, "_afd_suppress_metadata_send", False)): + if self._afd_suppress_metadata_send: return dp_metadata = forward_context.dp_metadata ubatch_slices = forward_context.ubatch_slices @@ -1074,7 +1357,7 @@ def _install_async_moe_ubatch_metadata_on_forward_context( def _send_dp_metadata( self, dp_metadata: DPMetadata | AFDDPMetadata | None, - ubatch_slices: Any, + ubatch_slices: UBatchSlices | None, ) -> None: assert self.connector.control_plane is not None, ( "_send_dp_metadata needs control plane driven connectors" @@ -1128,11 +1411,10 @@ def _build_capture_dp_metadata(self, num_tokens: int) -> DPMetadata | AFDDPMetad dp_size = int(self.vllm_config.parallel_config.data_parallel_size) return _make_uniform_dp_metadata(dp_size, int(num_tokens)) - def load_model(self, *args: Any, **kwargs: Any) -> Any: - result = super().load_model(*args, **kwargs) + def load_model(self) -> None: + super().load_model() if bool(self.vllm_config.parallel_config.use_ubatching): self._install_ascend_ubatch_wrapper() - return result def _install_ascend_ubatch_wrapper(self) -> None: if isinstance(self.model, AscendUBatchWrapper): @@ -1151,13 +1433,13 @@ def _install_ascend_ubatch_wrapper(self) -> None: self.device, ) - def get_model(self) -> Any: + def get_model(self) -> nn.Module: if isinstance(self.model, AscendUBatchWrapper): return self.model.unwrap() return super().get_model() - def initialize_attn_backend(self, *args: Any, **kwargs: Any) -> Any: - result = super().initialize_attn_backend(*args, **kwargs) + def initialize_attn_backend(self, kv_cache_config: KVCacheConfig) -> None: + super().initialize_attn_backend(kv_cache_config) if ( bool( self.vllm_config.parallel_config.use_ubatching, @@ -1165,7 +1447,6 @@ def initialize_attn_backend(self, *args: Any, **kwargs: Any) -> Any: or self.afd_async_extra_info.async_moe_ubatching ): self._ensure_two_metadata_builders() - return result def _ensure_two_metadata_builders(self) -> None: for attn_groups in self.attn_groups: @@ -1178,7 +1459,7 @@ def _ensure_two_metadata_builders(self) -> None: num_metadata_builders=2, ) - def _sync_metadata_across_dp( + def _sync_afd_metadata_across_dp( self, num_tokens_unpadded: int, num_tokens_padded: int | None = None, @@ -1193,16 +1474,11 @@ def _sync_metadata_across_dp( num_tokens_padded = num_tokens_unpadded if self.dp_size == 1: - moe_comm_type = select_moe_comm_method( - num_tokens_padded, - self.vllm_config, - ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, num_tokens_padded, uniform_decode=uniform_decode, vllm_config=self.vllm_config, - moe_comm_type=moe_comm_type, ) return should_ubatch, num_tokens_padded, None, cudagraph_mode @@ -1212,16 +1488,11 @@ def _sync_metadata_across_dp( device="cpu", dtype=torch.int32, ) - moe_comm_type = select_moe_comm_method( - num_tokens_padded, - self.vllm_config, - ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, num_tokens_padded, uniform_decode=uniform_decode, vllm_config=self.vllm_config, - moe_comm_type=moe_comm_type, ) return ( should_ubatch, @@ -1235,26 +1506,18 @@ def _sync_metadata_across_dp( self.vllm_config, is_draft_model, ) - may_ubatch = bool( - getattr(parallel_config, "enable_dbo", False) - and getattr(parallel_config, "use_ubatching", False) - ) + may_ubatch = bool(parallel_config.enable_dbo and parallel_config.use_ubatching) if can_skip_dp_sync and not may_ubatch: num_tokens_after_padding = torch.tensor( [num_tokens_padded] * self.dp_size, device="cpu", dtype=torch.int32, ) - moe_comm_type = select_moe_comm_method( - num_tokens_padded, - self.vllm_config, - ) should_ubatch = check_enable_ubatch( num_tokens_unpadded, num_tokens_padded, uniform_decode=uniform_decode, vllm_config=self.vllm_config, - moe_comm_type=moe_comm_type, ) return ( should_ubatch, @@ -1274,16 +1537,11 @@ def _sync_metadata_across_dp( min_tokens_across_dp = int(num_tokens_unpadded_across_dp.min().item()) synced_cudagraph_mode = CUDAGraphMode(int(packed_tensor[-1, :].min().item())) - moe_comm_type = select_moe_comm_method( - max_tokens_across_dp, - self.vllm_config, - ) should_ubatch = check_enable_ubatch( min_tokens_across_dp, max_tokens_across_dp, uniform_decode=uniform_decode, vllm_config=self.vllm_config, - moe_comm_type=moe_comm_type, ) if allow_dp_padding or is_draft_model or should_ubatch: @@ -1301,6 +1559,14 @@ def _sync_metadata_across_dp( synced_cudagraph_mode, ) + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner._determine_batch_execution_and_padding. + # Patch reason: upstream intentionally leaves NPU microbatching disabled and + # uses its native DP synchronization, which cannot coordinate AFD stages. + # Patch functionality: retain the upstream signature and execution/padding + # logic while enabling microbatching only during AFD live execution and using + # the AFD control-plane-aware DP synchronization path. + # Signature: matches upstream; no added parameters or changed defaults. def _determine_batch_execution_and_padding( self, num_tokens: int, @@ -1308,7 +1574,7 @@ def _determine_batch_execution_and_padding( num_scheduled_tokens_np: np.ndarray, max_num_scheduled_tokens: int, use_cascade_attn: bool, - allow_microbatching: bool = True, + allow_microbatching: bool = False, force_eager: bool = False, force_uniform_decode: bool | None = None, force_has_lora: bool | None = None, @@ -1372,15 +1638,18 @@ def dispatch_cudagraph( ) should_ubatch, num_tokens_across_dp = False, None + # ### PATCH START: AFD DP metadata synchronization if self.vllm_config.parallel_config.data_parallel_size > 1: should_ubatch, _, num_tokens_across_dp, synced_cudagraph_mode = ( - self._sync_metadata_across_dp( + self._sync_afd_metadata_across_dp( num_tokens_unpadded=num_tokens, num_tokens_padded=num_tokens_padded, uniform_decode=uniform_decode, cudagraph_mode=cudagraph_mode, allow_dp_padding=(cudagraph_mode != CUDAGraphMode.NONE) - or enable_sp(self.vllm_config), + or enable_sp(self.vllm_config) + or oproj_tp_enable() + or embedding_tp_enable(), ) ) if num_tokens_across_dp is not None: @@ -1392,19 +1661,17 @@ def dispatch_cudagraph( ) assert batch_descriptor.num_tokens == num_tokens_padded else: - moe_comm_type = select_moe_comm_method( - num_tokens_padded, - self.vllm_config, - ) should_ubatch = check_enable_ubatch( num_tokens, num_tokens_padded, uniform_decode=uniform_decode, vllm_config=self.vllm_config, - moe_comm_type=moe_comm_type, ) - if not allow_microbatching: + # ### PATCH END: AFD DP metadata synchronization + # ### PATCH START: AFD live NPU microbatching + if not (allow_microbatching or self._afd_live_execution): should_ubatch = False + # ### PATCH END: AFD live NPU microbatching cudagraph_stats = None if self.vllm_config.observability_config.cudagraph_metrics: @@ -1422,18 +1689,25 @@ def dispatch_cudagraph( cudagraph_stats, ) + # Upstream source: vllm-ascend commit 80d8c194f, + # NPUModelRunner.sync_and_slice_intermediate_tensors. + # Patch reason: upstream sizes PP intermediate tensors from the combined + # token count, which is too small when SP rounds each AFD ubatch separately. + # Patch functionality: compute the sum of per-ubatch SP slices and grow the + # reusable intermediate buffer before copying or returning that slice. + # Signature: matches upstream; no added parameters. def sync_and_slice_intermediate_tensors( self, num_tokens: int, - intermediate_tensors: Any | None, + intermediate_tensors: IntermediateTensors | None, sync_self: bool, - ) -> Any: + ) -> IntermediateTensors: assert self.intermediate_tensors is not None tp = self.vllm_config.parallel_config.tensor_parallel_size - if self.ubatch_slices is None: - slice_len = (num_tokens + tp - 1) // tp if enable_sp() else num_tokens - else: + slice_len = (num_tokens + tp - 1) // tp if enable_sp() else num_tokens + if self.ubatch_slices is not None: + # ### PATCH START: AFD per-ubatch intermediate slice and buffer slice_len = ( sum( (ubatch_slice.num_tokens + tp - 1) // tp @@ -1451,18 +1725,29 @@ def sync_and_slice_intermediate_tensors( dtype=self.dtype, device=self.device, ) + # ### PATCH END: AFD per-ubatch intermediate slice and buffer if sync_self: assert intermediate_tensors is not None + # ### PATCH START: AFD intermediate copy length copy_len = slice_len + # ### PATCH END: AFD intermediate copy length for k, v in intermediate_tensors.items(): + if k not in self.intermediate_tensors.tensors: + base_tensor = self.intermediate_tensors["hidden_states"] + self.intermediate_tensors[k] = v.new_empty( + (base_tensor.shape[0], *v.shape[1:]), + ) self.intermediate_tensors[k][:copy_len].copy_( v[:copy_len], non_blocking=True, ) - return IntermediateTensors( + # ### PATCH START: AFD intermediate output slice + result = IntermediateTensors( {k: v[:slice_len] for k, v in self.intermediate_tensors.items()}, ) + # ### PATCH END: AFD intermediate output slice + return result def shutdown(self) -> None: stop_afd_npu_profiler(self.prof) @@ -1490,37 +1775,20 @@ def _dp_metadata_debug_key( ) -> tuple[tuple[int, tuple]]: key_parts: list[tuple[int, tuple]] = [] for stage_idx, metadata in sorted(dp_metadata_list.items()): - values = metadata.num_tokens_across_dp_cpu - tolist = getattr(values, "tolist", None) - if callable(tolist): - values = tolist() - elif hasattr(values, "item"): - values = [values.item()] - try: - values_tuple = tuple(int(value) for value in values) - except TypeError: - values_tuple = (int(values),) + values_tuple = tuple( + int(value) for value in metadata.num_tokens_across_dp_cpu.tolist() + ) key_parts.append((int(stage_idx), values_tuple)) return tuple(key_parts) -def _attention_metadata_values( - args: tuple[Any, ...], - kwargs: dict[str, Any], -) -> dict[str, Any]: - values = dict(zip(_ATTENTION_METADATA_ARG_NAMES, args, strict=False)) - values.update(kwargs) - return values - - def _normalize_metadata_ubatch_slices( - ubatch_slices: Any, - values: dict[str, Any], -) -> Any: + ubatch_slices: UBatchSlices | None, + num_tokens_padded: int | None, + num_reqs_padded: int | None, +) -> UBatchSlices | None: if not ubatch_slices: return ubatch_slices - num_tokens_padded = values.get("num_tokens_padded") - num_reqs_padded = values.get("num_reqs_padded") if num_tokens_padded is None or num_reqs_padded is None: return ubatch_slices @@ -1537,60 +1805,4 @@ def _normalize_metadata_ubatch_slices( ) -def _replace_attention_metadata_ubatch_slices( - args: tuple[Any, ...], - kwargs: dict[str, Any], - ubatch_slices: Any, -) -> tuple[tuple[Any, ...], dict[str, Any]]: - ubatch_index = _ATTENTION_METADATA_ARG_NAMES.index("ubatch_slices") - if len(args) > ubatch_index: - new_args = list(args) - new_args[ubatch_index] = ubatch_slices - return tuple(new_args), kwargs - new_kwargs = dict(kwargs) - new_kwargs["ubatch_slices"] = ubatch_slices - return args, new_kwargs - - -def _model_forward_values( - args: tuple[Any, ...], - kwargs: dict[str, Any], -) -> tuple[Any, Any, Any, Any, Any, dict[str, Any]]: - names = [ - "num_tokens_padded", - "input_ids", - "positions", - "intermediate_tensors", - "inputs_embeds", - ] - values = dict(zip(names, args, strict=False)) - model_kwargs = dict(kwargs) - for name in names: - if name in model_kwargs: - values[name] = model_kwargs.pop(name) - return ( - values["num_tokens_padded"], - values.get("input_ids"), - values.get("positions"), - values.get("intermediate_tensors"), - values.get("inputs_embeds"), - model_kwargs, - ) - - -_ATTENTION_METADATA_ARG_NAMES = [ - "num_tokens", - "num_reqs", - "max_query_len", - "num_tokens_padded", - "num_reqs_padded", - "ubatch_slices", - "logits_indices", - "use_spec_decode", - "for_cudagraph_capture", - "num_scheduled_tokens", - "num_scheduled_tokens_np", - "cascade_attn_prefix_lens", -] - __all__ = ["AFDNPUAttentionModelRunner"] diff --git a/afd_plugin/v1/worker/npu/forward_context.py b/afd_plugin/v1/worker/npu/forward_context.py index 522acaea..01dc6d68 100644 --- a/afd_plugin/v1/worker/npu/forward_context.py +++ b/afd_plugin/v1/worker/npu/forward_context.py @@ -39,6 +39,12 @@ def create_ascend_forward_context( build_ubatch_afd_metadata(afd_metadata, ubatch_slices, ubatch_num), ) + ubatch_slice = ubatch_slices[ubatch_num] + is_padding = ( + cur_forward_context.is_padding[ubatch_slice.token_slice] + if cur_forward_context.is_padding is not None + else None + ) new_forward_context = ForwardContext( no_compile_layers=vllm_config.compilation_config.static_forward_context, all_moe_layers=cur_forward_context.all_moe_layers, @@ -50,9 +56,9 @@ def create_ascend_forward_context( ubatch_slices=ubatch_slices, skip_compiled=skip_compiled, additional_kwargs=parent_kwargs, + is_padding=is_padding, ) - ubatch_slice = ubatch_slices[ubatch_num] num_tokens = ubatch_slice.num_tokens tp_world_size = get_tensor_model_parallel_world_size() dp_world_size = get_dp_group().world_size @@ -70,7 +76,6 @@ def create_ascend_forward_context( new_forward_context.flash_comm_v1_enabled = ( cur_forward_context.flash_comm_v1_enabled ) - new_forward_context.flashcomm_v2_enabled = cur_forward_context.flashcomm_v2_enabled new_forward_context.pad_size = 0 new_forward_context.is_first_layer = cur_forward_context.is_first_layer new_forward_context.layer_idx = cur_forward_context.layer_idx @@ -89,21 +94,20 @@ def create_ascend_forward_context( new_forward_context.max_tokens_across_pcp = ( cur_forward_context.max_tokens_across_pcp ) + new_forward_context.sinks = cur_forward_context.sinks + new_forward_context.input_ids = cur_forward_context.input_ids + new_forward_context.eplb_heat_collection_status = ( + cur_forward_context.eplb_heat_collection_status + ) - if ( - new_forward_context.flash_comm_v1_enabled - or new_forward_context.flashcomm_v2_enabled - ): + if new_forward_context.flash_comm_v1_enabled: new_forward_context.pad_size = ( tp_world_size - (num_tokens % tp_world_size) ) % tp_world_size if dp_world_size > 1 and dp_metadata is not None: - max_tokens_across_dp = dp_metadata.max_tokens_across_dp_cpu.item() - if ( - new_forward_context.flash_comm_v1_enabled - or new_forward_context.flashcomm_v2_enabled - ): + max_tokens_across_dp = dp_metadata.num_tokens_across_dp_cpu.max().item() + if new_forward_context.flash_comm_v1_enabled: padded_length = ( (max_tokens_across_dp + tp_world_size - 1) // tp_world_size @@ -118,7 +122,7 @@ def create_ascend_forward_context( new_forward_context.padded_num_tokens = ( math.ceil(max_tokens_across_dp / tp_world_size) * tp_world_size ) - cur_mc2_mask = getattr(cur_forward_context, "mc2_mask", None) + cur_mc2_mask = cur_forward_context.mc2_mask if cur_mc2_mask is not None: mc2_mask = torch.zeros( (new_forward_context.padded_num_tokens,), diff --git a/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py b/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py index d0632511..16bb8c7a 100644 --- a/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py +++ b/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py @@ -9,6 +9,7 @@ import threading from collections.abc import Callable from dataclasses import dataclass +from typing import cast import torch import torch_npu # noqa: F401 @@ -36,6 +37,69 @@ make_ubatch_contexts, ) +AFD_NPU_NUM_UBATCHES = 2 +_READY_BARRIER_PARTIES = AFD_NPU_NUM_UBATCHES + 1 +AscendLastRankOutput = torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]] +AscendModelOutput = AscendLastRankOutput | IntermediateTensors + + +def _cat_ubatch_outputs( + sorted_results: list[AscendLastRankOutput], +) -> AscendLastRankOutput: + """Preserve the current Ascend model-output structure across ubatches. + + Upstream source: vLLM commit 68b0c3135, + ``gpu_ubatch_wrapper._cat_ubatch_outputs``. Ascend auxiliary hidden states + use ``tuple[Tensor, list[Tensor]]`` rather than upstream's tuple of tensors, + so this plugin-owned wrapper concatenates that concrete nested contract. + """ + assert sorted_results + first_result = sorted_results[0] + # ### PATCH START: Ascend auxiliary hidden-state output + if isinstance(first_result, tuple): + tuple_results = cast( + list[tuple[torch.Tensor, list[torch.Tensor]]], + sorted_results, + ) + num_aux_outputs = len(first_result[1]) + assert all(len(result[1]) == num_aux_outputs for result in tuple_results) + return ( + torch.cat([result[0] for result in tuple_results], dim=0), + [ + torch.cat( + [result[1][index] for result in tuple_results], + dim=0, + ) + for index in range(num_aux_outputs) + ], + ) + # ### PATCH END: Ascend auxiliary hidden-state output + return torch.cat(cast(list[torch.Tensor], sorted_results), dim=0) + + +def _all_gather_ubatch_output( + output: AscendLastRankOutput, + pad_size: int, +) -> AscendLastRankOutput: + if isinstance(output, tuple): + hidden_states, aux_hidden_states = output + gathered_hidden_states = _all_gather_ubatch_output(hidden_states, pad_size) + assert isinstance(gathered_hidden_states, torch.Tensor) + gathered_aux_hidden_states = [ + _all_gather_ubatch_output(aux_hidden_state, pad_size) + for aux_hidden_state in aux_hidden_states + ] + assert all( + isinstance(aux_hidden_state, torch.Tensor) + for aux_hidden_state in gathered_aux_hidden_states + ) + return gathered_hidden_states, cast( + list[torch.Tensor], + gathered_aux_hidden_states, + ) + output = tensor_model_parallel_all_gather(output, 0) + return output[:-pad_size, :] if pad_size > 0 else output + @dataclass class AscendUbatchMetadata(UbatchMetadata): @@ -47,7 +111,7 @@ class AscendUbatchMetadata(UbatchMetadata): class AscendNPUGraphMetaData: aclgraph: torch.npu.NPUGraph ubatch_metadata: list[AscendUbatchMetadata] - outputs: torch.Tensor | IntermediateTensors | None = None + outputs: AscendModelOutput | None = None class AscendUBatchWrapper(UBatchWrapper): @@ -64,7 +128,8 @@ def __init__( self.vllm_config = vllm_config self.compilation_config = vllm_config.compilation_config self.comm_stream = torch.npu.Stream(device=device) - self.ready_barrier = threading.Barrier(3) + assert self.vllm_config.parallel_config.num_ubatches == AFD_NPU_NUM_UBATCHES + self.ready_barrier = threading.Barrier(_READY_BARRIER_PARTIES) self.cudagraphs: dict[int, AscendNPUGraphMetaData] = {} self.cudagraph_wrapper = None if runtime_mode is not CUDAGraphMode.NONE: @@ -112,6 +177,8 @@ def __call__(self, *args, **kwargs): assert self.cudagraph_wrapper is not None return self.cudagraph_wrapper(*args, **kwargs) + assert len(ubatch_slices) == AFD_NPU_NUM_UBATCHES + attn_metadata = forward_context.attn_metadata num_tokens = sum(ubatch_slice.num_tokens for ubatch_slice in ubatch_slices) input_ids = kwargs["input_ids"] @@ -163,6 +230,7 @@ def __call__(self, *args, **kwargs): cudagraph_metadata = self.cudagraphs[num_tokens] cudagraph_metadata.aclgraph.replay() get_forward_context().dbo_enabled = True + assert cudagraph_metadata.outputs is not None return cudagraph_metadata.outputs ubatch_metadata = self._make_ubatch_metadata( @@ -300,20 +368,21 @@ def _merge_intermediate_tensors(self, intermediate_tensor_list): def _merge_outputs( self, - sorted_results: list[torch.Tensor | IntermediateTensors], + sorted_results: list[AscendModelOutput], ubatch_metadata: list[AscendUbatchMetadata], - ) -> torch.Tensor | IntermediateTensors: + ) -> AscendModelOutput: if not get_pp_group().is_last_rank: - return self._merge_intermediate_tensors(sorted_results) + return self._merge_intermediate_tensors( + cast(list[IntermediateTensors], sorted_results), + ) + last_rank_results = cast(list[AscendLastRankOutput], sorted_results) ubatch_forward_context = ubatch_metadata[0].context.forward_context if ubatch_forward_context.flash_comm_v1_enabled: - for i, result in enumerate(sorted_results): - sorted_results[i] = tensor_model_parallel_all_gather(result, 0) + for i, result in enumerate(last_rank_results): pad_size = ubatch_metadata[i].context.forward_context.pad_size - if pad_size > 0: - sorted_results[i] = sorted_results[i][:-pad_size, :] - return torch.cat(sorted_results, dim=0) + last_rank_results[i] = _all_gather_ubatch_output(result, pad_size) + return _cat_ubatch_outputs(last_rank_results) @torch.inference_mode() def _run_ubatch_thread(self, results, model, ubatch_metadata): @@ -330,8 +399,8 @@ def _run_ubatches( self, ubatch_metadata: list[AscendUbatchMetadata], model, - ) -> torch.Tensor | IntermediateTensors: - results: list[tuple[int, torch.Tensor | IntermediateTensors]] = [] + ) -> AscendModelOutput: + results: list[tuple[int, AscendModelOutput]] = [] with override_forward_context(None): ubatch_threads = [] for metadata in ubatch_metadata: @@ -354,8 +423,8 @@ def _capture_ubatches( self, ubatch_metadata: list[AscendUbatchMetadata], model, - ) -> torch.Tensor | IntermediateTensors: - results: list[tuple[int, torch.Tensor | IntermediateTensors]] = [] + ) -> AscendModelOutput: + results: list[tuple[int, AscendModelOutput]] = [] compute_stream = ubatch_metadata[0].context.compute_stream num_tokens = sum(metadata.num_tokens for metadata in ubatch_metadata) @@ -389,11 +458,13 @@ def _capture_ubatches( ) self.cudagraphs[num_tokens] = cudagraph_metadata get_forward_context().dbo_enabled = True + assert cudagraph_metadata.outputs is not None return cudagraph_metadata.outputs __all__ = [ "AscendNPUGraphMetaData", + "AscendModelOutput", "AscendUBatchWrapper", "AscendUbatchMetadata", ] diff --git a/afd_plugin/v1/worker/npu/ubatch_utils.py b/afd_plugin/v1/worker/npu/ubatch_utils.py index 331eb1ca..55ab6a55 100644 --- a/afd_plugin/v1/worker/npu/ubatch_utils.py +++ b/afd_plugin/v1/worker/npu/ubatch_utils.py @@ -2,8 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """Ascend ubatch helpers owned by the AFD plugin. -Copied from vLLM-Ascend commit cdd212830271249a1cafcb850c210133f21771c5; -kept plugin-owned so AFD retains DBO support independent of upstream changes. +Originally copied from vLLM-Ascend commit +``cdd212830271249a1cafcb850c210133f21771c5`` and aligned with the attention +metadata schema at commit ``80d8c194f``. It remains plugin-owned because the +current vLLM-Ascend release no longer provides NPU ubatch helpers. """ import numpy as np @@ -14,7 +16,6 @@ UBatchSlices, check_ubatch_thresholds, ) -from vllm_ascend.ascend_forward_context import MoECommType from vllm_ascend.attention.utils import AscendCommonAttentionMetadata @@ -29,8 +30,8 @@ def is_last_ubatch_empty( def _cp_enabled(vllm_config: VllmConfig) -> bool: parallel_config = vllm_config.parallel_config return ( - getattr(parallel_config, "prefill_context_parallel_size", 1) > 1 - or getattr(parallel_config, "decode_context_parallel_size", 1) > 1 + parallel_config.prefill_context_parallel_size > 1 + or parallel_config.decode_context_parallel_size > 1 ) @@ -39,10 +40,9 @@ def check_enable_ubatch( num_tokens_padded: int, uniform_decode: bool, vllm_config: VllmConfig, - moe_comm_type: MoECommType | None, ) -> bool: parallel_config = vllm_config.parallel_config - num_ubatches = getattr(parallel_config, "num_ubatches", 2) + num_ubatches = parallel_config.num_ubatches if num_ubatches != 2: return False if num_tokens_padded < num_ubatches: @@ -56,7 +56,7 @@ def check_enable_ubatch( num_tokens_unpadded, uniform_decode=uniform_decode, ) - if not getattr(parallel_config, "enable_dbo", False): + if not parallel_config.enable_dbo: return False if not should_attempt_ubatching: return False @@ -151,7 +151,7 @@ def maybe_create_ubatch_slices( if not should_ubatch: return None, None - num_ubatches = getattr(vllm_config.parallel_config, "num_ubatches", 2) + num_ubatches = vllm_config.parallel_config.num_ubatches assert num_ubatches == 2, "Ascend ubatching currently supports exactly 2 ubatches." split_point = int(num_tokens_padded) // num_ubatches @@ -279,11 +279,65 @@ def _make_metadata_with_slice( causal=attn_metadata.causal, num_input_tokens=num_actual_tokens, actual_seq_lengths_q=actual_seq_lengths_q, - positions=attn_metadata.positions[token_slice], + positions=( + attn_metadata.positions[:, token_slice] + if attn_metadata.positions.ndim == 2 + else attn_metadata.positions[token_slice] + ), + positions_cpu=( + ( + attn_metadata.positions_cpu[:, token_slice] + if attn_metadata.positions_cpu.ndim == 2 + else attn_metadata.positions_cpu[token_slice] + ) + if attn_metadata.positions_cpu is not None + else None + ), attn_state=attn_metadata.attn_state, graph_pad_size=attn_metadata.graph_pad_size, decode_token_per_req=attn_metadata.decode_token_per_req, - kvcomp_metadata=attn_metadata.kvcomp_metadata, + dcp_local_seq_lens=( + attn_metadata.dcp_local_seq_lens[request_slice] + if attn_metadata.dcp_local_seq_lens is not None + else None + ), + dcp_local_seq_lens_cpu=( + attn_metadata.dcp_local_seq_lens_cpu[request_slice] + if attn_metadata.dcp_local_seq_lens_cpu is not None + else None + ), + is_prefilling=( + attn_metadata.is_prefilling[request_slice] + if attn_metadata.is_prefilling is not None + else None + ), + seq_lens_cpu_upper_bound=( + attn_metadata.seq_lens_cpu_upper_bound[request_slice] + if attn_metadata.seq_lens_cpu_upper_bound is not None + else None + ), + mm_req_doc_ranges=attn_metadata.mm_req_doc_ranges, + rswa_prefix_lens=( + attn_metadata.rswa_prefix_lens[request_slice] + if attn_metadata.rswa_prefix_lens is not None + else None + ), + context_parallel_metadata=attn_metadata.context_parallel_metadata, + group_len=( + attn_metadata.group_len[request_slice] + if attn_metadata.group_len is not None + else None + ), + group_key_idx=( + attn_metadata.group_key_idx[request_slice] + if attn_metadata.group_key_idx is not None + else None + ), + group_key_cache_idx=( + attn_metadata.group_key_cache_idx[request_slice] + if attn_metadata.group_key_cache_idx is not None + else None + ), ) metadata.encoder_seq_lens = ( attn_metadata.encoder_seq_lens[request_slice] diff --git a/tests/unit/compat/patches/test_config_validation.py b/tests/unit/compat/patches/test_config_validation.py index f6b418e3..2a74e73d 100644 --- a/tests/unit/compat/patches/test_config_validation.py +++ b/tests/unit/compat/patches/test_config_validation.py @@ -136,19 +136,27 @@ def test_config_validation_patch_allows_vllm_dev_checkout(monkeypatch): assert cfg.parallel_config.all2all_backend == "allgather_reducescatter" -def test_config_validation_patch_relaxes_repeated_vllm_post_init(monkeypatch): +def test_config_validation_patch_selects_worker_after_upstream_post_init(monkeypatch): + arg_utils_module, _config_module = _install_fake_vllm_config(monkeypatch) + _load_patch_module() + args = _engine_args(active=True) + + cfg = arg_utils_module.EngineArgs.create_engine_config(args) + assert cfg.post_init_backend == "deepep_low_latency" + assert cfg.parallel_config.all2all_backend == "allgather_reducescatter" + assert cfg.parallel_config.worker_cls == ATTENTION_WORKER_FQCN + + +def test_config_validation_patch_relaxes_explicit_post_init_revalidation(monkeypatch): arg_utils_module, _config_module = _install_fake_vllm_config(monkeypatch) _load_patch_module() args = _engine_args(active=True) cfg = arg_utils_module.EngineArgs.create_engine_config(args) - cfg.additional_config = args.additional_config - cfg.parallel_config.use_ubatching = True cfg.__post_init__() assert cfg.post_init_backend == "deepep_low_latency" assert cfg.parallel_config.all2all_backend == "allgather_reducescatter" - assert cfg.parallel_config.worker_cls == ATTENTION_WORKER_FQCN @pytest.mark.parametrize( @@ -230,6 +238,30 @@ def test_config_validation_patch_auto_selects_without_ubatching(monkeypatch): assert cfg.parallel_config.worker_cls == FFN_WORKER_FQCN +def test_config_validation_installs_ascend_patch_only_on_npu(monkeypatch): + arg_utils_module, config_module = _install_fake_vllm_config(monkeypatch) + import afd_plugin.compat.npu as npu_compat + + calls = [] + monkeypatch.setattr( + npu_compat, + "apply_afd_ascend_patches_if_needed", + lambda: calls.append("npu"), + ) + patch_module = _load_patch_module() + importlib.reload(patch_module) + + cuda_args = _engine_args(active=True) + arg_utils_module.EngineArgs.create_engine_config(cuda_args) + assert calls == [] + + config_module.VllmConfig.platform_worker_cls = VLLM_ASCEND_NPU_WORKER_FQCN + _set_fake_platform(is_cuda=False, device_type="npu") + npu_args = _engine_args(active=True) + arg_utils_module.EngineArgs.create_engine_config(npu_args) + assert calls == ["npu"] + + def test_config_validation_patch_preserves_non_afd_platform_default(monkeypatch): arg_utils_module, config_module = _install_fake_vllm_config(monkeypatch) config_module.VllmConfig.platform_worker_cls = VLLM_GPU_WORKER_FQCN diff --git a/tests/unit/compat/test_runtime.py b/tests/unit/compat/test_runtime.py index 9c74a90b..9809440b 100644 --- a/tests/unit/compat/test_runtime.py +++ b/tests/unit/compat/test_runtime.py @@ -6,6 +6,8 @@ from contextlib import contextmanager from types import ModuleType, SimpleNamespace +import pytest + from afd_plugin.compat.npu import runtime as ascend_runtime from afd_plugin.compat.npu.runtime import fix_all2all_backend_for_afd @@ -136,6 +138,7 @@ class FakeParallelConfig: def __init__(self, *, enable_dbo, ubatch_size): self.enable_dbo = enable_dbo self.ubatch_size = ubatch_size + self.all2all_backend = "deepep_low_latency" @property def use_ubatching(self): @@ -147,7 +150,14 @@ def _fix_incompatible_config(vllm_config): parallel_config = vllm_config.parallel_config parallel_config.enable_dbo = False parallel_config.ubatch_size = 0 - return "fixed" + + @classmethod + def check_and_update_config(cls, vllm_config): + cls._fix_incompatible_config(vllm_config) + parallel_config = vllm_config.parallel_config + parallel_config.all2all_backend = "flashinfer_all2allv" + if getattr(vllm_config, "fail_update", False): + raise RuntimeError("upstream config failure") def afd_vllm_config(*, active=True): config = _vllm_config() @@ -172,12 +182,22 @@ def afd_vllm_config(*, active=True): ascend_runtime.apply_afd_ascend_patches_if_needed() config = afd_vllm_config() - assert NPUPlatform._fix_incompatible_config(config) == "fixed" + assert NPUPlatform.check_and_update_config(config) is None assert config.parallel_config.enable_dbo is True assert config.parallel_config.use_ubatching is True assert config.parallel_config.ubatch_size == 4 + assert config.parallel_config.all2all_backend == "deepep_low_latency" + + failing_config = afd_vllm_config() + failing_config.fail_update = True + with pytest.raises(RuntimeError, match="upstream config failure"): + NPUPlatform.check_and_update_config(failing_config) + assert failing_config.parallel_config.enable_dbo is True + assert failing_config.parallel_config.ubatch_size == 4 + assert failing_config.parallel_config.all2all_backend == "deepep_low_latency" inactive_config = afd_vllm_config(active=False) - assert NPUPlatform._fix_incompatible_config(inactive_config) == "fixed" + assert NPUPlatform.check_and_update_config(inactive_config) is None assert inactive_config.parallel_config.enable_dbo is False assert inactive_config.parallel_config.use_ubatching is False + assert inactive_config.parallel_config.all2all_backend == "flashinfer_all2allv" diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index df418115..41e106cf 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -232,6 +232,111 @@ def _new_attention_runner(): return object.__new__(AFDNPUAttentionModelRunner) +def test_npu_attention_live_execution_scope_restores_on_success_and_error( + monkeypatch, +): + _require_npu_runtime() + from afd_plugin.v1.worker.npu import attention_model_runner + + runner = _new_attention_runner() + runner.prof = None + runner._afd_live_execution = False + observed_live_state = [] + + monkeypatch.setattr( + attention_model_runner, + "step_afd_npu_profiler", + lambda _prof: None, + ) + + def execute_success(_runner, _scheduler_output, _intermediate_tensors=None): + observed_live_state.append(_runner._afd_live_execution) + return "success" + + monkeypatch.setattr( + attention_model_runner.NPUModelRunner, + "execute_model", + execute_success, + ) + assert runner.execute_model(object()) == "success" + assert observed_live_state == [True] + assert runner._afd_live_execution is False + + def execute_failure(_runner, _scheduler_output, _intermediate_tensors=None): + observed_live_state.append(_runner._afd_live_execution) + raise RuntimeError("execute failure") + + monkeypatch.setattr( + attention_model_runner.NPUModelRunner, + "execute_model", + execute_failure, + ) + with pytest.raises(RuntimeError, match="execute failure"): + runner.execute_model(object()) + assert observed_live_state == [True, True] + assert runner._afd_live_execution is False + + +def test_npu_attention_non_live_execution_disables_microbatching(monkeypatch): + _require_npu_runtime() + import numpy as np + from vllm.config import CUDAGraphMode + from vllm.forward_context import BatchDescriptor + + from afd_plugin.v1.worker.npu import attention_model_runner + + runner = _new_attention_runner() + runner._afd_live_execution = False + runner._pad_for_sequence_parallelism = lambda num_tokens: num_tokens + runner.input_batch = SimpleNamespace( + num_computed_tokens_cpu=np.ones(4, dtype=np.int32), + lora_id_to_lora_request={}, + ) + runner.speculative_config = None + runner.uniform_decode_query_len = 1 + runner.model_config = SimpleNamespace(is_encoder_decoder=False) + runner.vllm_config = SimpleNamespace( + parallel_config=SimpleNamespace( + data_parallel_size=1, + tensor_parallel_size=1, + ), + observability_config=SimpleNamespace(cudagraph_metrics=False), + ) + runner.cudagraph_dispatcher = SimpleNamespace( + dispatch=lambda **kwargs: ( + CUDAGraphMode.NONE, + BatchDescriptor(kwargs["num_tokens"]), + ), + ) + monkeypatch.setattr(attention_model_runner, "enable_sp", lambda _config: False) + monkeypatch.setattr( + attention_model_runner, + "check_enable_ubatch", + lambda *_args, **_kwargs: True, + ) + + result = runner._determine_batch_execution_and_padding( + num_tokens=4, + num_reqs=4, + num_scheduled_tokens_np=np.ones(4, dtype=np.int32), + max_num_scheduled_tokens=1, + use_cascade_attn=False, + allow_microbatching=False, + ) + assert result[2] is False + + runner._afd_live_execution = True + result = runner._determine_batch_execution_and_padding( + num_tokens=4, + num_reqs=4, + num_scheduled_tokens_np=np.ones(4, dtype=np.int32), + max_num_scheduled_tokens=1, + use_cascade_attn=False, + allow_microbatching=False, + ) + assert result[2] is True + + def _new_ffn_runner(): _require_npu_runtime() from afd_plugin.v1.worker.npu.ffn_model_runner import AFDNPUFFNModelRunner @@ -600,7 +705,6 @@ def test_npu_create_ascend_forward_context_marks_current_ubatch(monkeypatch): capturing=False, mmrs_fusion=False, flash_comm_v1_enabled=False, - flashcomm_v2_enabled=False, is_first_layer=True, layer_idx=0, prefetch_mlp_gate_up_proj=False, @@ -610,6 +714,10 @@ def test_npu_create_ascend_forward_context_marks_current_ubatch(monkeypatch): is_draft_model_prefill=False, draft_attn_metadatas=None, max_tokens_across_pcp=None, + sinks=False, + input_ids=None, + eplb_heat_collection_status=False, + is_padding=None, mc2_mask=None, ) ubatch_slices = [ @@ -1139,6 +1247,64 @@ def test_npu_feature_validation_allows_two_ubatches_only(): fail_if_unsupported_npu_afd_features(config) +def test_npu_ubatch_output_merge_preserves_aux_hidden_states(): + _require_npu_runtime() + import torch + + from afd_plugin.v1.worker.npu.npu_ubatch_wrapper import _cat_ubatch_outputs + + merged = _cat_ubatch_outputs( + [ + (torch.tensor([[1.0]]), [torch.tensor([[2.0]])]), + (torch.tensor([[3.0]]), [torch.tensor([[4.0]])]), + ], + ) + + assert isinstance(merged, tuple) + assert merged[0].tolist() == [[1.0], [3.0]] + assert len(merged[1]) == 1 + assert merged[1][0].tolist() == [[2.0], [4.0]] + + +def test_npu_ubatch_all_gather_preserves_aux_outputs_and_trims_padding( + monkeypatch, +): + _require_npu_runtime() + import torch + + from afd_plugin.v1.worker.npu import npu_ubatch_wrapper + + gathered_inputs = [] + + def fake_all_gather(output, dim): + assert dim == 0 + gathered_inputs.append(output.clone()) + return torch.cat((output, output + 10), dim=0) + + monkeypatch.setattr( + npu_ubatch_wrapper, + "tensor_model_parallel_all_gather", + fake_all_gather, + ) + output = ( + torch.tensor([[1.0], [2.0]]), + [ + torch.tensor([[3.0], [4.0]]), + torch.tensor([[5.0], [6.0]]), + ], + ) + + gathered = npu_ubatch_wrapper._all_gather_ubatch_output(output, pad_size=1) + + assert isinstance(gathered, tuple) + assert gathered[0].tolist() == [[1.0], [2.0], [11.0]] + assert [tensor.tolist() for tensor in gathered[1]] == [ + [[3.0], [4.0], [13.0]], + [[5.0], [6.0], [15.0]], + ] + assert len(gathered_inputs) == 3 + + def test_npu_async_feature_validation_requires_async_config_and_eager(): with pytest.raises(RuntimeError, match="async=true"): fail_if_unsupported_npu_afd_features( @@ -1242,7 +1408,7 @@ def test_npu_async_moe_ubatching_validation_requires_supported_shape(): ) -def test_npu_ubatch_allows_mc2_comm_when_thresholds_are_met(monkeypatch): +def test_npu_ubatch_enabled_when_thresholds_are_met(monkeypatch): fake_numpy = ModuleType("numpy") fake_numpy.ndarray = object fake_torch = ModuleType("torch") @@ -1268,11 +1434,6 @@ def check_ubatch_thresholds(config, num_tokens, uniform_decode): fake_vllm_ascend = ModuleType("vllm_ascend") fake_forward_context = ModuleType("vllm_ascend.ascend_forward_context") - class MoECommType: - MC2 = object() - FUSED_MC2 = object() - - fake_forward_context.MoECommType = MoECommType fake_attention = ModuleType("vllm_ascend.attention") fake_attention_utils = ModuleType("vllm_ascend.attention.utils") fake_attention_utils.AscendCommonAttentionMetadata = object @@ -1319,14 +1480,12 @@ class MoECommType: num_tokens_padded=12, uniform_decode=True, vllm_config=config, - moe_comm_type=ubatch_utils.MoECommType.MC2, ) assert ubatch_utils.check_enable_ubatch( num_tokens_unpadded=12, num_tokens_padded=12, uniform_decode=True, vllm_config=config, - moe_comm_type=ubatch_utils.MoECommType.FUSED_MC2, ) finally: sys.modules.pop(module_name, None) From 02345c015f84f1f2bf6c878630c2e825048dd850 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 3 Aug 2026 09:48:08 +0800 Subject: [PATCH 05/10] Fix NPU force load balance formatting Signed-off-by: jiangkuaixue123 --- afd_plugin/compat/patches/npu/force_load_balance.py | 1 + tests/unit/compat/patches/test_force_load_balance.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/afd_plugin/compat/patches/npu/force_load_balance.py b/afd_plugin/compat/patches/npu/force_load_balance.py index 4e0a1259..c7e85097 100644 --- a/afd_plugin/compat/patches/npu/force_load_balance.py +++ b/afd_plugin/compat/patches/npu/force_load_balance.py @@ -438,6 +438,7 @@ def apply( final_hidden_states += zero_expert_result return final_hidden_states + AscendW8A8DynamicFusedMoEMethod.__init__ = __init__ AscendW8A8DynamicFusedMoEMethod.apply = apply diff --git a/tests/unit/compat/patches/test_force_load_balance.py b/tests/unit/compat/patches/test_force_load_balance.py index 6662942d..e355b978 100644 --- a/tests/unit/compat/patches/test_force_load_balance.py +++ b/tests/unit/compat/patches/test_force_load_balance.py @@ -128,9 +128,7 @@ def get_moe_num_logical_experts( num_logical_experts = getattr(layer.moe_config, "num_logical_experts", None) if num_logical_experts is not None: return int(num_logical_experts) - return int( - num_experts - global_redundant_expert_num - num_shared_experts - ) + return int(num_experts - global_redundant_expert_num - num_shared_experts) methods_base_mod.get_moe_num_logical_experts = get_moe_num_logical_experts w8a8_mod = types.ModuleType("vllm_ascend.quantization.methods.w8a8_dynamic") From 8adb00f80657b5f94aaf9ede79f9aebec7017632 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 3 Aug 2026 10:37:51 +0800 Subject: [PATCH 06/10] Refresh v0.26 documentation Signed-off-by: jiangkuaixue123 --- .github/ISSUE_TEMPLATE/100-bug-report.yml | 4 +- .../ISSUE_TEMPLATE/200-feature-request.yml | 4 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- README.md | 45 +++++++++---------- afd_plugin/connectors/README.md | 5 +++ .../module/compatibility_and_patches.md | 18 ++++---- docs/design/module/connector_contracts.md | 8 ++-- docs/design/module/execution_platforms.md | 23 +++++----- docs/design/module/index.md | 6 +-- docs/design/module/model_integration.md | 23 ++++++---- docs/design/module/plugin_boundary.md | 4 +- docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md | 9 +++- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 19 +++++--- docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md | 6 +-- recipe/README.md | 22 ++++----- .../deepseek_v2_lite/README.md | 4 +- .../deepseek_v3_2/README.md | 10 ++++- .../deepseek_v3_2/README.md | 12 ++--- 18 files changed, 128 insertions(+), 96 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/100-bug-report.yml b/.github/ISSUE_TEMPLATE/100-bug-report.yml index a4f8b144..a5de3994 100644 --- a/.github/ISSUE_TEMPLATE/100-bug-report.yml +++ b/.github/ISSUE_TEMPLATE/100-bug-report.yml @@ -8,7 +8,7 @@ body: attributes: value: > Before submitting, please search existing issues to avoid duplicates. - afd-plugin targets vLLM v0.19.1 unless an issue explicitly says otherwise. + afd-plugin targets vLLM v0.26.0 unless an issue explicitly says otherwise. - type: markdown attributes: @@ -76,5 +76,5 @@ body: options: - label: I searched existing issues for related reports. required: true - - label: I confirmed whether this reproduces against vLLM v0.19.1 or explained why not. + - label: I confirmed whether this reproduces against vLLM v0.26.0 or explained why not. required: true diff --git a/.github/ISSUE_TEMPLATE/200-feature-request.yml b/.github/ISSUE_TEMPLATE/200-feature-request.yml index d7f27e2b..1ca3be12 100644 --- a/.github/ISSUE_TEMPLATE/200-feature-request.yml +++ b/.github/ISSUE_TEMPLATE/200-feature-request.yml @@ -8,7 +8,7 @@ body: attributes: value: > Before submitting, please search existing issues and RFCs. Keep proposals - tied to vLLM v0.19.1 compatibility unless the issue explicitly proposes a + tied to vLLM v0.26.0 compatibility unless the issue explicitly proposes a version expansion. - type: textarea @@ -31,7 +31,7 @@ body: id: compatibility attributes: label: vLLM compatibility and extension points - description: Explain how this should work without modifying the vLLM v0.19.1 source tree. + description: Explain how this should work without modifying the vLLM v0.26.0 source tree. placeholder: | Preferred extension point: Compat shim needed: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 72ca8af4..4aa892c6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -42,7 +42,7 @@ or any behavior that intentionally differs from the original AFD commit. - [ ] Purpose is clear and linked to public context when possible. - [ ] Scope is bounded. -- [ ] Compatibility with vLLM v0.19.1 is considered. +- [ ] Compatibility with vLLM v0.26.0 is considered. - [ ] No changes are made to the vLLM source checkout. - [ ] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [ ] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. diff --git a/README.md b/README.md index fd4fd746..657581ee 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ tests for GPU and Ascend NPU deployments. > This project is still experimental and needs more large-scale testing across > different hardware backends. -The target runtime is **vLLM `v0.19.1`**. The plugin does not modify the vLLM +The target runtime is **vLLM `v0.26.0`**. The plugin does not modify the vLLM source tree. AFD behavior is installed through the `vllm.general_plugins` entry point, `--additional-config`, automatically selected role workers, plugin-owned model wrappers, and narrow version-scoped compatibility shims. @@ -34,6 +34,9 @@ Core runtime support: execution for CUDA and Ascend NPU. - Eager and `FULL_DECODE_ONLY` graph execution, plus backend-specific profiling support. +- Native DBO with exactly two ubatches on CUDA and the synchronous Ascend path. +- DeepSeek MoE handoff at the remote-experts boundary on CUDA, with the gate + placed on either Attention or FFN. Model support: @@ -49,7 +52,7 @@ See the [recipe index](recipe/README.md) for deployment and benchmark examples. | --- | --- | --- | --- | --- | --- | | `P2pNcclAFDConnector` | CUDA | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | FFN ranks are ordered before Attention ranks. `num_attention_ranks` must be greater than or equal to `num_ffn_ranks` and divisible by it. See the [DeepSeek V2 Lite recipe](recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md). | | `CAMP2pAFDConnector` | Ascend NPU | Decode | Sync | `FULL_DECODE_ONLY` ACL graph | Uses HCCL/CAMP2P custom ops. Ascend ops build by default on NPU platforms. See the [synchronous DeepSeek V3.2 recipe](recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md). | -| `CAMAsyncAFDConnector` | Ascend NPU | Prefill | Async | Not supported | Uses CAM async-DP custom ops and requires `async=true` with the Ascend NPU workers. See the [asynchronous DeepSeek V3.2 recipe](recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md). | +| `CAMAsyncAFDConnector` | Ascend NPU | Prefill | Async | Not supported | Experimental. The v0.26 upgrade did not revalidate this path; the checked-in [PCP8 recipe](recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) records the earlier v0.19.1 experiment only. | Connector implementations are grouped by backend package: `afd_plugin.connectors.gpu` for GPU-only connectors, @@ -57,11 +60,13 @@ Connector implementations are grouped by backend package: Known gaps: -- vLLM versions other than `0.19.1` are not claimed as supported. +- vLLM versions other than `0.26.0` are not claimed as supported. - vLLM/vLLM-Ascend model runner v2 is not supported. - GPU and NPU E2E tests are opt-in and require real hardware plus model weights. - GPU CUDA graph support is limited to `FULL_DECODE_ONLY`. -- GPU DBO plus CUDA graph is limited to exactly two ubatches. +- Native DBO is limited to exactly two ubatches. +- CAM async and PCP-based NPU model-runner-v1 deployments are not part of the + v0.26 validated runtime matrix. ## Install @@ -85,36 +90,28 @@ command: uv sync --group dev --extra vllm ``` -The optional extra pins `vllm==0.19.1`. +The optional extra pins `vllm==0.26.0`. ### Ascend NPU installation -AFD's Ascend path is validated on openEuler 22.03 (aarch64) with Ascend 910C / -Atlas A3. Install a compatible driver and firmware, and confirm the devices -with `npu-smi info`. Use this compatible release baseline: +AFD's synchronous Ascend path is validated on openEuler 22.03 (aarch64) with +Ascend 910C / Atlas A3. Install a compatible driver and firmware, and confirm +the devices with `npu-smi info`. Use this source baseline: | Component | Version | | --- | --- | | Python | `3.10` or `3.11` | -| vLLM | `0.19.1` | -| vLLM-Ascend | `0.19.1rc1` | -| CANN / NNAL | `8.5.1` | -| torch | `2.9.0` | -| torch-npu | `2.9.0` | +| vLLM | `0.26.0` | +| vLLM-Ascend | commit [`80d8c194f`](https://github.com/vllm-project/vllm-ascend/commit/80d8c194f7584b17fe08065ea99a130916f6b0e7) | +| CANN / torch / torch-npu | Use the mutually compatible versions required by that vLLM-Ascend source snapshot. | #### Environment -The following A3/openEuler environment has been validated: - -```bash -docker pull quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler -``` - -Use the fixed -[vLLM-Ascend installation guide](https://github.com/vllm-project/vllm-ascend/blob/v0.19.1rc1/docs/source/installation.md) -to start the image with the device and driver configuration for your host. The -image includes the matched CANN and NNAL environment. Run the remaining commands -inside the container from the AFD repository root. +The v0.26 integration was refreshed against vLLM-Ascend commit `80d8c194f`; +the repository does not currently claim a released v0.26 container tag. Use the +[installation guide at that source snapshot](https://github.com/vllm-project/vllm-ascend/blob/80d8c194f7584b17fe08065ea99a130916f6b0e7/docs/source/installation.md) +to prepare a matching A3/openEuler environment, then install AFD from the +repository root. Do not reuse the former v0.19.1rc1 image as a v0.26 runtime. #### Install AFD diff --git a/afd_plugin/connectors/README.md b/afd_plugin/connectors/README.md index 5cf132a3..dd040523 100644 --- a/afd_plugin/connectors/README.md +++ b/afd_plugin/connectors/README.md @@ -8,5 +8,10 @@ AFD connector implementations are grouped by backend: by `afd_plugin.connectors.npu.camp2p`, and `CAMAsyncAFDConnector` is implemented by `afd_plugin.connectors.npu.async_cam`. +The vLLM 0.26 support matrix validates GPU `P2pNcclAFDConnector` and synchronous +NPU `CAMP2pAFDConnector`. `CAMAsyncAFDConnector` remains experimental and was +not revalidated during the v0.26 upgrade; its PCP8 recipe is a historical +v0.19.1 experiment. + Shared connector contracts, metadata containers, factory registration, and backend-neutral helpers stay in `afd_plugin.connectors`. diff --git a/docs/design/module/compatibility_and_patches.md b/docs/design/module/compatibility_and_patches.md index 8224a139..91b487f3 100644 --- a/docs/design/module/compatibility_and_patches.md +++ b/docs/design/module/compatibility_and_patches.md @@ -36,7 +36,7 @@ verified_platform_refs: related_issues: - "#86" - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # Compatibility and patches @@ -56,17 +56,17 @@ patch modules must not become a general home for AFD-owned functionality. ## Supported upstream boundary -The package extra pins the supported vLLM release, and +The package extra pins vLLM `0.26.0`, and [`compat/vllm.py`](../../../afd_plugin/compat/vllm.py) enforces the same target. Direct strict calls raise for a missing or different vLLM; plugin registration calls the check with `strict=False`, so it warns and continues. This warning policy does not make another vLLM release supported. -The Ascend patch sources name current vLLM-Ascend modules and are validated by -the recorded Ascend environment, but the repository does not declare an exact -vLLM-Ascend package dependency in `pyproject.toml`. Before this document can be -normative, maintainers must record the source tag/commit used to refresh every -copied Ascend function, not only the container image that exercised it. +The NPU v0.26 refresh is based on vLLM-Ascend commit +[`80d8c194f`](https://github.com/vllm-project/vllm-ascend/commit/80d8c194f7584b17fe08065ea99a130916f6b0e7). +The repository does not declare a vLLM-Ascend package dependency in +`pyproject.toml`, so this source commit and the recorded NPU validation are the +compatibility evidence rather than a released package or container tag. ## Implementation evidence @@ -108,7 +108,7 @@ not the package dependency policy. | [`async_dp_forward_context.py`](../../../afd_plugin/compat/patches/async_dp_forward_context.py): `vllm.forward_context.set_forward_context` plus already-imported worker aliases | Skips native `DPMetadata` construction/coordination only for AFD async-DP; otherwise uses the copied upstream flow. | Imported by `register_afd`; same target/dev/unknown guard. Rebinds known already-imported aliases so callers do not retain the old function. | [`test_async_dp_forward_context.py`](../../../tests/unit/compat/patches/test_async_dp_forward_context.py) covers async skip and non-async coordination. | Remove when vLLM supports a per-engine-role opt-out from native MoE DP metadata coordination. | | [`config_validation.py`](../../../afd_plugin/compat/patches/config_validation.py): `EngineArgs.create_engine_config`, `VllmConfig.__post_init__` | For AFD-owned ubatching with a non-DeepEP backend, temporarily presents `deepep_low_latency` during upstream validation and restores the configured backend. After upstream platform normalization, maps an initial `worker_cls="auto"` to the role-specific CUDA or standard Ascend AFD worker. | Imported by `register_afd`; accepts the target version, development versions, or missing version metadata. Saves originals on upstream modules under AFD-specific attributes before installing wrappers. Explicit worker paths and non-AFD configs are not remapped. | [`test_config_validation.py`](../../../tests/unit/compat/patches/test_config_validation.py) covers backend relaxation, four role/platform mappings, explicit and non-AFD preservation, repeated validation, unsupported platforms, and dev versions. | Remove the backend branch when upstream validation distinguishes plugin-owned ubatching; remove worker mapping when vLLM offers plugin-owned role-aware worker selection. | | [`engine_core.py`](../../../afd_plugin/compat/patches/engine_core.py): `EngineCore.__init__`, `_initialize_kv_caches`, `shutdown`; `EngineCoreProc.run_busy_loop`; `DPEngineCoreProc.run_busy_loop` | AFD FFN becomes a connector daemon: construct executor, skip scheduler/KV setup, return an empty KV-shaped result on late paths, start/monitor/stop the FFN worker loop, and use FFN-safe shutdown. Non-FFN branches copy pinned upstream behavior. | Imported by `register_afd`; **no patch-local version guard and no saved-original sentinel**. Direct class assignment means the package pin and review discipline are the compatibility guard. | [`test_engine_core.py`](../../../tests/unit/compat/patches/test_engine_core.py) covers FFN initialization, non-FFN behavior, and daemon start/stop; role runtime tests cover error propagation. | Remove when vLLM offers a headless connector-daemon engine lifecycle or an executor mode that does not require scheduler/KV ownership. | -| [`npu/ascend_platform.py`](../../../afd_plugin/compat/patches/npu/ascend_platform.py): `NPUPlatform._fix_incompatible_config` | Snapshots AFD DBO state, runs upstream normalization, and restores configured `enable_dbo`/`ubatch_size` when AFD needs them; non-AFD behavior is unchanged. | Called through `apply_afd_ascend_patches_if_needed`; no version guard. Saves the original on the class and uses both a class sentinel and runtime-facade sentinel. Missing vLLM-Ascend is a no-op. | [`test_runtime.py`](../../../tests/unit/compat/test_runtime.py) and [`test_npu_runtime.py`](../../../tests/unit/v1/worker/test_npu_runtime.py). | Remove when vLLM-Ascend recognizes plugin-owned DBO workers or no longer clears these fields. | +| [`npu/ascend_platform.py`](../../../afd_plugin/compat/patches/npu/ascend_platform.py): `NPUPlatform.check_and_update_config` | Snapshots AFD DBO state, runs upstream normalization, and restores configured `enable_dbo`, `ubatch_size`, and `all2all_backend` in `finally`; non-AFD behavior is unchanged. | Called through `apply_afd_ascend_patches_if_needed`; no version guard. Saves the original on the class and uses a class sentinel. The runtime facade caches success only after the wrapper is installed, so an early missing vLLM-Ascend import remains retryable. | [`test_runtime.py`](../../../tests/unit/compat/test_runtime.py) and [`test_npu_runtime.py`](../../../tests/unit/v1/worker/test_npu_runtime.py). | Remove when vLLM-Ascend recognizes plugin-owned DBO workers or no longer clears these fields. | | [`npu/force_load_balance.py`](../../../afd_plugin/compat/patches/npu/force_load_balance.py): `AscendFusedMoE.__init__`, `AscendW8A8DynamicFusedMoEMethod.apply` | Adds AFD profiling configuration and replaces routed expert IDs with a deterministic balanced buffer only when the layer-owned switch is enabled; normal model-selected routing remains unchanged. This switch changes outputs and is not a correctness feature. | Imported only when vLLM-Ascend is discoverable; **no patch-local version guard or explicit reload sentinel**. Functions copy the current upstream bodies with marked AFD deltas. | [`test_force_load_balance.py`](../../../tests/unit/compat/patches/test_force_load_balance.py) covers buffer bounds, determinism, growth, override, and pass-through. | Upstream a deterministic expert-routing profiling hook in vLLM-Ascend, then delete both copied functions. | ## Non-patch compatibility adapters @@ -197,7 +197,7 @@ The inventory covers current production patch files, but the document remains - `engine_core` and force-load-balance lack patch-local version/idempotence guards; - best-effort grouped imports can produce a partially applied patch set; -- the exact vLLM-Ascend source tag/commit is not recorded in package metadata; +- the exact vLLM-Ascend source commit is documented but not pinned as package metadata; - owners have not approved the candidate invariants as normative contracts. Runtime refactor decisions in diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index 6b108d8d..6a754f98 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -34,7 +34,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # Connector contracts @@ -94,8 +94,10 @@ than an `AFDConfig` field. Unknown fields fail in the selected connector parser. | `CAMAsyncAFDConnector` | `dynamicQuant`, `attn_ranks_per_dp`, `async_moe_ubatching`, `async_moe_num_ubatches`, and `async_moe_split`. Runtime validation further limits dynamic quantization and the optional request-boundary pipeline. | The common `compute_gate_on_attention` field remains on `AFDConfig` and is the -model-routing selector. CAMP2P also parses a connector-local field with that -name for its operator contract; both paths currently reject enabling it. +model-routing selector. CUDA P2P supports both values. CAMP2P also parses a +connector-local field with that name for its operator contract, but the current +synchronous NPU runtime requires both common and connector-local values to be +`false`. CAM async requires the common field to be `true`. ## Current connector modes diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index aa058d63..fc4c5dff 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -49,7 +49,7 @@ verified_platform_refs: related_issues: - "#86" - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # Execution platforms @@ -287,17 +287,17 @@ an expansion of the supported runtime contract. | Platform/path | Execution | Ubatching | Routing/quantization limits | Evidence | | --- | --- | --- | --- | --- | -| CUDA + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` CUDA Graph | Native DBO, exactly two ubatches | Role-aware DeepSeek path; P2P topology validated by connector/config tests | GPU serving, graph, TP, profiler, model, and accuracy E2E tests | +| CUDA + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` CUDA Graph | Native DBO, exactly two ubatches | DeepSeek remote-experts boundary; Attention-side or FFN-side gate; EPLB rejected on the Attention remote-experts role | GPU serving, graph, TP/EP, DP/EP, DBO, profiler, model, and accuracy E2E tests | | Ascend + `CAMP2pAFDConnector` | Eager or current ACL Graph path | Native DBO, exactly two ubatches | Common and connector-local `compute_gate_on_attention=false`; `connector_extra_config.quant_mode=0`; plugin CANN ops required | NPU serving, graph, TP, ops, profiler, model, and accuracy E2E tests | -| Ascend + `CAMAsyncAFDConnector` | Eager only | Native DBO rejected; optional async MoE ubatching uses exactly two request-boundary stages | `async=true`; documented path uses common `compute_gate_on_attention=true`; decode context parallel unsupported for async MoE ubatching; `connector_extra_config.dynamicQuant` is 0 or 1; external CAM ops required | Async CAM connector unit tests and `test_async_cam_npu.py` | +| Ascend + `CAMAsyncAFDConnector` | Eager only | Native DBO rejected; optional async MoE ubatching uses exactly two request-boundary stages | Experimental code path; the former PCP8 recipe is incompatible with v0.26 model runner v1 and was not revalidated in this upgrade | Unit coverage only for the retained connector/model adapters; no v0.26 hardware support claim | -All paths use the supported vLLM release and model runner v1. GPU/NPU rank -topology and connector resource rules remain owned by +The validated CUDA and synchronous Ascend paths use vLLM 0.26.0 and model +runner v1. GPU/NPU rank topology and connector resource rules remain owned by [connector contracts](connector_contracts.md). -The repository does not record a canonical CUDA container. Current Ascend -guides record the tested container; this is test evidence, not an authoritative -vLLM-Ascend package or tag pin. +The repository does not record a canonical CUDA container or a released +vLLM-Ascend v0.26 container. The NPU implementation records source commit +`80d8c194f`; environment evidence is not an authoritative package tag. ## Failure and cleanup boundaries @@ -322,14 +322,15 @@ both platforms. ## Upstream relationship and validation requirements CUDA behavior is developed against the pinned vLLM release. The recorded -Ascend container is test evidence, not an authoritative package/tag pin. Build, +Ascend source snapshot and environment are compatibility evidence, not a +released package/tag pin. Build, graph, profiler, and native-op changes require the matching unit and hardware E2E paths listed above. ## Limitations and open issues -The official vLLM-Ascend tag/container and canonical CUDA/Ascend versus GPU/NPU -terminology are unresolved. This document uses CUDA/Ascend for backend +The official vLLM-Ascend v0.26 tag/container and canonical CUDA/Ascend versus +GPU/NPU terminology are unresolved. This document uses CUDA/Ascend for backend mechanisms and preserves GPU/NPU where it appears in public names, environment variables, or test markers. See [#129](https://github.com/JiusiServe/afd-plugin/issues/129). diff --git a/docs/design/module/index.md b/docs/design/module/index.md index 441040c6..64f55d06 100644 --- a/docs/design/module/index.md +++ b/docs/design/module/index.md @@ -17,14 +17,14 @@ validation_paths: - "tests/unit/**" - "tests/e2e/**" upstream_refs: - - "vLLM" - - "vLLM-Ascend environment evidence recorded in the NPU guides" + - "vLLM 0.26.0" + - "vLLM-Ascend commit 80d8c194f and environment evidence recorded in the NPU guides" verified_platform_refs: - "CUDA: tests/e2e tests marked gpu; no canonical image is recorded" - "Ascend E2E environment recorded in the installation and NPU guides" related_issues: - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # AFD module design diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 66467b24..3ccd7e9a 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -31,7 +31,7 @@ related_issues: - "#88" - "#105" - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # Model integration @@ -54,6 +54,7 @@ make a backend-specific worker class the shared model API. | --- | --- | --- | | Registration map | [`afd_plugin/__init__.py`](../../../afd_plugin/__init__.py) | [`test_package.py`](../../../tests/unit/package/test_package.py) | | Role-aware model and weight loading | [`deepseek_v2.py`](../../../afd_plugin/model_executor/models/deepseek_v2.py) | [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py), model and accuracy E2E suites | +| CUDA remote-experts boundary | [`deepseek_v2.py`](../../../afd_plugin/model_executor/models/deepseek_v2.py), [`gpu/p2p.py`](../../../afd_plugin/connectors/gpu/p2p.py) | [`test_p2p_experts_contract.py`](../../../tests/unit/connectors/test_p2p_experts_contract.py), [`test_deepseek_v2_proxy.py`](../../../tests/unit/model_executor/models/test_deepseek_v2_proxy.py) | | Forward-context adapter | [`forward_context.py`](../../../afd_plugin/model_executor/models/forward_context.py) | [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | | Ascend Attention-side gate | [`npu/deepseek_v2_attention_gate.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py) | Attention-gate unit cases in [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | | Ascend CAM orchestration | [`npu/deepseek_v2_async_cam_forward.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py) | Async/ubatch unit cases and [`test_async_cam_npu.py`](../../../tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py) | @@ -89,14 +90,19 @@ needed by the split execution. | Layer/component | Attention role | FFN role | | --- | --- | --- | | Attention module and KV-facing computation | Constructed and executed. | Not constructed. | -| MoE or dense MLP, normal mode | Not constructed; output is sent after post-Attention normalization. | Constructed and executed from connector input. | -| MoE gate with `compute_gate_on_attention=true` | Constructed; produces router logits and top-k payloads before send. | Expert MLP is constructed; it consumes routed payloads without rerunning the gate. | +| MoE with `compute_gate_on_attention=false` | CUDA constructs the native MoE shell with a parameter-free internal-router experts proxy; NPU sends after post-Attention normalization. | Native gate and experts are constructed and executed from connector input. | +| MoE with `compute_gate_on_attention=true` | CUDA keeps the native gate and uses an external-router experts proxy; NPU uses its Attention-side gate helper. | Expert MLP is constructed and consumes transferred router logits or routed payloads without rerunning the gate. | +| Dense MLP, normal mode | Not constructed; output is sent after post-Attention normalization. | Constructed and executed from connector input. | | Dense MLP with `compute_gate_on_attention=true` | Constructed and executed locally because there is no routed MoE handoff. | Not constructed and a dense-layer FFN compute request is rejected. | | Embedding, final norm, pipeline placeholders | Created according to the pinned pipeline-rank rules. | Same wrapper lifecycle rules; only role-required parameters are loaded. | -`compute_gate_on_attention` is rejected outside NPU before the split layer is -constructed. The current gate helper supports unquantized and Ascend W8A8 MoE -expert computation; unsupported quantization fails explicitly. +CUDA MoE always splits at the remote-experts boundary while preserving native +`DeepseekV2MoE.forward`. With gate-on-FFN, the proxy asks FFN to run its native +internal-router MoE. With gate-on-Attention, Attention runs the native gate and +FFN executes its external-router experts path. CUDA Attention-side remote +experts currently reject EPLB. The NPU gate helper supports unquantized and +Ascend W8A8 MoE expert computation; unsupported devices or quantization fail +explicitly. The full AFD model remains decorated with vLLM's compile support. Backend-only helpers are imported inside the NPU path so CUDA model import does not require @@ -203,9 +209,8 @@ the other role can be omitted from model/accuracy E2E coverage. not an implicit local-forward fallback. - AFD paths that require a connector, top-k payload, group list, or async stage metadata fail when that input is missing. -- Unsupported aux-hidden-state capture, non-NPU gate placement, unsupported - gate quantization, and inconsistent shared-expert dimensions fail - explicitly. +- Unsupported aux-hidden-state capture, unsupported device gate placement or + gate quantization, and inconsistent shared-expert dimensions fail explicitly. - The model owns modules, parameters, local intermediates, and layer computation. The runner owns forward-context installation and step lifecycle. The connector owns communication resources and transfer state. diff --git a/docs/design/module/plugin_boundary.md b/docs/design/module/plugin_boundary.md index e19f0c98..7b9525c9 100644 --- a/docs/design/module/plugin_boundary.md +++ b/docs/design/module/plugin_boundary.md @@ -34,7 +34,7 @@ verified_platform_refs: related_issues: - "#89" - "#129" -last_reviewed: 2026-07-20 +last_reviewed: 2026-08-03 --- # Plugin boundary @@ -133,7 +133,7 @@ must be placed under `connector_extra_config`. | `host`, `port` | `127.0.0.1`, `1239` | Connector rendezvous/control endpoint inputs. | | `num_attention_ranks`, `num_ffn_ranks` | `1`, `1` | AFD role-group sizes used by topology construction. | | `afd_role_rank` | `0` | Rank within the selected role group. | -| `compute_gate_on_attention` | `false` | Moves supported gate/MoE routing work to Attention; current implementation is NPU-only. | +| `compute_gate_on_attention` | `false` | Moves supported gate/MoE routing work to Attention. CUDA supports both gate placements at the remote-experts boundary; synchronous CAMP2P still requires `false`, while CAM async requires `true`. | | `connector_extra_config` | `{}` | Envelope key parsed by the selected connector into a typed `ConnectorExtraInfo`; it is not stored on `AFDConfig`. | The compatibility aliases `afd_connector`, `afd_role`, `afd_port`, `afd_host`, diff --git a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md index 98b4fdaa..569ec465 100644 --- a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md @@ -4,7 +4,11 @@ P2pNcclAFDConnector (implemented with vLLM's PyNcclCommunicator) is a GPU-backed ## When to use `P2pNcclAFDConnector` -Use this connector for CUDA deployments that disaggregate Attention and FFN workers and exchange hidden states synchronously through NCCL point-to-point communication. +Use this connector for CUDA deployments that disaggregate Attention and FFN +workers and exchange hidden states synchronously through NCCL point-to-point +communication. On vLLM 0.26, DeepSeek MoE layers keep the native MoE forward +contract and replace only the local experts with an AFD remote-experts proxy. +The MoE gate may run on Attention or FFN. It supports both prefill and decode which all support eager mode. CUDA graph support is currently limited to `FULL_DECODE_ONLY`, which is mainly used in decode instance. The checked-in DeepSeek V2 Lite recipes cover colocated and prefill/decode-disaggregated deployments. @@ -88,7 +92,7 @@ AFD configuration is supplied through vLLM's `--additional-config` under the `af | `num_attention_ranks` | `int` | `1` | Total number of AFD Attention ranks, including DP/TP-derived worker ranks. Must be positive. | | `num_ffn_ranks` | `int` | `1` | Total number of AFD FFN ranks, including DP/TP-derived worker ranks. Must be positive. | | `afd_role_rank` | `int` | `0` | Rank within the selected role group. Must satisfy `0 <= rank < num__ranks`. Runners normally derive it from DP/PCP/TP placement; users should not assign duplicate role ranks. | -| `compute_gate_on_attention` | `bool` | `false` | Must be `false`. Whether Attention computes MoE gate outputs before sending work to FFN. This is a general AFD field, not a PyNccl transport setting. | +| `compute_gate_on_attention` | `bool` | `false` | When `false`, FFN owns the native gate and experts. When `true`, Attention owns the native gate and transfers router logits to the FFN external-router expert path. | | `connector_extra_config` | `dict` | `{}` | Must remain empty; `P2pNcclAFDConnector` does not currently support connector-specific options. | | `async` / `async_dp` | `bool` | `false` | Must remain `false` for `P2pNcclAFDConnector`; AFD async mode requires `CAMAsyncAFDConnector`. | @@ -161,6 +165,7 @@ For complete `1A1F`, `2A2F`, `4A4F`, eager, DBO, and CUDA graph examples, see `r - The rendezvous base port and derived subgroup ports must be free and reachable. - Initialization is collective: missing ranks, mismatched counts, or duplicate role ranks can cause initialization failure or timeout. - Current GPU CUDA graph support is `FULL_DECODE_ONLY`; GPU DBO plus CUDA graph is limited to exactly two ubatches. +- CUDA remote experts do not currently support EPLB on the Attention role. - To enable DBO, set `--enable-dbo`, and configure the threshold with `--dbo-decode-token-threshold` and `--dbo-prefill-token-threshold`. See `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite` for examples. - The repository recipes currently validate specific GPU layouts (including DeepSeek V2 Lite and tested A/H-class hardware). Cross-node use depends on NCCL/network configuration and is not established by the current recipes; document it as unverified rather than promising transparent fallback. - There is no automatic fallback from this connector to another transport. Select an NPU connector explicitly on Ascend. diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index 5ec5276b..f1d5523b 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -8,12 +8,21 @@ through CAM async dispatch/combine operators. This guide describes the supported deployment shape, configuration contract, rank mapping, data flow, startup requirements, and current limitations. The [DeepSeek-V3.2 recipe](../../recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) -contains the complete validated multi-node launch commands. +contains the historical multi-node launch commands and measurements. + +> [!WARNING] +> The vLLM 0.26 upgrade did not revalidate CAM async. The linked PCP8 recipe and +> its measurements were produced with the former vLLM/vLLM-Ascend 0.19.1 +> environment. vLLM-Ascend 0.26 removes PCP from model runner v1, so those +> commands are retained as historical experiment records, not as a supported +> v0.26 deployment recipe. Current v0.26 support claims cover the synchronous +> `CAMP2pAFDConnector` path. ## When to use this connector -Use `CAMAsyncAFDConnector` for the currently supported asynchronous Ascend NPU -prefill path when all of the following are true: +The retained implementation describes an asynchronous Ascend NPU prefill path +with the following constraints. These are code-level constraints, not a v0.26 +hardware support claim: - CAM operator packages are installed on every node; - Attention performs MoE gating before dispatch to FFN ranks; @@ -195,10 +204,10 @@ current async MoE metadata path does not support it. ## Requirements -The checked-in recipe has been verified with: +The historical checked-in recipe was verified with: - Ascend 910C; -- `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler`; +- `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler` (legacy only); - the included `CAM_ascend910_93_openEuler_aarch64.run` installer; - `umdk_cam_op_lib-208.1.0b1-cp311-cp311-linux_aarch64.whl`. diff --git a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md index 305e2ef6..f2fe2e1a 100644 --- a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md @@ -16,8 +16,8 @@ is limited to `FULL_DECODE_ONLY`. ## Prerequisites -- An Ascend PyTorch, vLLM, and vLLM Ascend environment compatible with the - versions pinned by this repository. +- vLLM `0.26.0` and an Ascend PyTorch/vLLM-Ascend environment based on source + commit [`80d8c194f`](https://github.com/vllm-project/vllm-ascend/commit/80d8c194f7584b17fe08065ea99a130916f6b0e7). - The AFD Ascend custom operators must be built and available at runtime. - HCCL connectivity for the data path and Gloo connectivity for DP metadata. - Identical model hidden size, model dtype, AFD topology, rendezvous address, @@ -122,7 +122,7 @@ Pass AFD configuration through vLLM's `--additional-config` option under the | `port` | `int` | `1239` | AFD rendezvous port in `1..65535`. It is separate from the vLLM HTTP service ports. | | `num_attention_ranks` | `int` | `1` | Total number of Attention worker ranks. Must be positive. | | `num_ffn_ranks` | `int` | `1` | Total number of FFN worker ranks. Must be positive. | -| `afd_role_rank` | `int` | `0` | Base rank within the selected role. The worker normally derives each local role rank from its DP/PCP/TP placement. | +| `afd_role_rank` | `int` | `0` | Base rank within the selected role. The v0.26 NPU worker derives each local role rank from its DP/TP placement; PCP is not supported by model runner v1. | | `compute_gate_on_attention` | `bool` | `false` | Controls whether the MoE gate is computed on the Attention side or the FFN side. Currently only `false` is supported. | | `connector_extra_config` | `dict` | `{}` | CAMP2P-specific settings such as role-specific core counts and `quant_mode`. Unknown fields are rejected. | | `async` / `async_dp` | `bool` | `false` | Must remain `false` for the current synchronous; Ascend async mode requires `CAMAsyncAFDConnector`. | diff --git a/recipe/README.md b/recipe/README.md index 3ccb5fab..7dcd0b27 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -1,7 +1,9 @@ # AFD Recipes -This directory contains reproducible deployment and benchmark recipes for the -AFD connectors supported by this repository. +This directory contains deployment and benchmark recipes for AFD connectors. +Read each recipe's support status before running it: some historical experiment +records are retained for provenance but are not supported by the current +vLLM 0.26 runtime. ## Directory layout @@ -29,16 +31,16 @@ Directory names follow these conventions: ## Available recipes -| Hardware | Connector | Model | Recommended stage | Recipe | -| --- | --- | --- | --- | --- | -| GPU | `P2pNcclAFDConnector` | DeepSeek-V2-Lite | Decode | [Launch examples](gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md) | -| Ascend NPU | `CAMP2pAFDConnector` | DeepSeek-V3.2 | Decode | [Synchronous decode](npu/CAMP2pAFDConnector/deepseek_v3_2/README.md) | -| Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V3.2 | Prefill | [Asynchronous prefill](npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) | +| Hardware | Connector | Model | Recommended stage | v0.26 status | Recipe | +| --- | --- | --- | --- | --- | --- | +| GPU | `P2pNcclAFDConnector` | DeepSeek-V2-Lite | Decode | Validated | [Launch examples](gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md) | +| Ascend NPU | `CAMP2pAFDConnector` | DeepSeek-V3.2 | Decode | Validated | [Synchronous decode](npu/CAMP2pAFDConnector/deepseek_v3_2/README.md) | +| Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V3.2 | Prefill | Not revalidated; legacy PCP8 experiment | [Historical asynchronous prefill](npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) | Open the model-level README before running a recipe. It documents the required -hardware and container image, topology, environment variables, launch order, -and known limitations. Unless a recipe says otherwise, run its commands from -the repository root. +hardware and runtime baseline, topology, environment variables, launch order, +support status, and known limitations. Unless a recipe says otherwise, run its +commands from the repository root. ## Adding a recipe diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md index 40ef132a..c00bb237 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md @@ -1,7 +1,7 @@ # DeepSeek-V2-Lite AFD Examples End-to-end launch scripts for running DeepSeek-V2-Lite with the AFD -(Attention-FFN Disaggregation) plugin on vLLM `v0.19.1`. +(Attention-FFN Disaggregation) plugin on vLLM `v0.26.0`. > [!NOTE] > `P2pNcclAFDConnector` is an example connector implementation. Contributions @@ -12,7 +12,7 @@ End-to-end launch scripts for running DeepSeek-V2-Lite with the AFD - Install [NIXL](https://github.com/ai-dynamo/nixl). - At least 4 GPUs(A/H-class, tested against L20X). -- vLLM `v0.19.1` and the `afd-plugin` package installed in the same +- vLLM `v0.26.0` and the `afd-plugin` package installed in the same environment (see repository root `AGENTS.md`). - DeepSeek-V2-Lite weights on disk. All scripts default to `/path/model_weights/DeepSeek-V2-Lite`; override with diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md index 9df0970d..59c93db9 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md @@ -1,7 +1,13 @@ # CAMAsyncAFDConnector For DeepSeek-V3.2 Recipe -This recipe describes how to run DeepSeek-V3.2 with the AFD CAM async -connector on Ascend NPU. +> [!WARNING] +> This is a historical vLLM/vLLM-Ascend 0.19.1 PCP8 experiment. CAM async was +> not revalidated by the vLLM 0.26 upgrade, and vLLM-Ascend 0.26 removes PCP +> support from model runner v1. Keep these commands and measurements for +> provenance; do not treat them as a supported v0.26 deployment recipe. + +This recipe records how DeepSeek-V3.2 was run with the AFD CAM async connector +on Ascend NPU in the legacy environment. For the connector's complete configuration contract, rank derivation, data flow, native DBO distinction, and limitations, see the diff --git a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md index bd281798..d9027bc6 100644 --- a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md @@ -9,15 +9,15 @@ and FFN workers. ## Image and model requirements - Hardware: Ascend NPU, Atlas 900 A3 SuperPoD, 16 dies per node. -- Image: `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler`. +- Runtime: vLLM `0.26.0` with vLLM-Ascend source commit `80d8c194f`. - Model: DeepSeek-V3.2 with W8A8 weights. - AFD Plugin: install this repository in the container. -```bash -docker pull quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler -cd /path/to/afd-plugin -pip install -e . --no-build-isolation -v -``` +Prepare the matching A3/openEuler environment using the +[vLLM-Ascend installation guide at `80d8c194f`](https://github.com/vllm-project/vllm-ascend/blob/80d8c194f7584b17fe08065ea99a130916f6b0e7/docs/source/installation.md), +then install this repository with `pip install -e . --no-build-isolation -v`. +The former `v0.19.1rc1-a3-openeuler` image is not a supported runtime for this +v0.26 recipe. ## Topologies From 73bf449f93da35a9abec5471400035009dea1ed4 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Tue, 4 Aug 2026 23:26:57 +0800 Subject: [PATCH 07/10] fix(npu): upgrade CAM async for vLLM 0.26 Signed-off-by: jiangkuaixue123 --- afd_plugin/connectors/npu/async_cam.py | 70 ++++---- .../npu/deepseek_v2_async_cam_forward.py | 6 + .../models/npu/deepseek_v2_attention_gate.py | 44 +++-- .../v1/worker/npu/attention_model_runner.py | 5 +- afd_plugin/v1/worker/npu/ffn_model_runner.py | 60 ++++--- afd_plugin/v1/worker/npu/ffn_worker.py | 44 +++-- .../v1/worker/npu/npu_ubatch_wrapper.py | 2 +- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 16 +- .../deepseek_v3_2/README.md | 24 ++- .../deepseek_v2_lite/test_async_cam_npu.py | 133 +++++++++++---- tests/e2e/runner.py | 2 +- tests/e2e/test_runner.py | 34 ++++ .../connectors/test_async_cam_connector.py | 22 +-- .../models/test_forward_context.py | 62 +++++++ .../v1/worker/test_npu_device_contract.py | 16 ++ tests/unit/v1/worker/test_npu_runtime.py | 152 ++++++++++++++++++ 16 files changed, 546 insertions(+), 146 deletions(-) create mode 100644 tests/unit/v1/worker/test_npu_device_contract.py diff --git a/afd_plugin/connectors/npu/async_cam.py b/afd_plugin/connectors/npu/async_cam.py index bc5195ab..c3c68cad 100644 --- a/afd_plugin/connectors/npu/async_cam.py +++ b/afd_plugin/connectors/npu/async_cam.py @@ -25,7 +25,6 @@ from __future__ import annotations -import inspect import os from collections.abc import Mapping from dataclasses import dataclass @@ -166,7 +165,7 @@ class AFDAsyncTransferState(AFDTransferState): layer_idx: int = 0 token_nums_rankid_layeridx: Tensor | None = None expert_token_nums_shared: Tensor | None = None - group_list: object = None + group_list: Tensor | None = None dynamic_scales: Tensor | None = None expand_x_shared: Tensor | None = None dynamic_scales_shared: Tensor | None = None @@ -313,16 +312,9 @@ def close(self) -> None: self._initialized = False def select_experts(self, **kwargs: Any) -> tuple[Tensor, Tensor]: - """Run the vLLM Ascend expert selector on the Attention side.""" + """Run the pinned vLLM-Ascend expert selector on Attention.""" from vllm_ascend.ops.fused_moe.experts_selector import select_experts - if "global_num_experts" in kwargs: - signature = inspect.signature(select_experts) - if ( - "global_num_experts" not in signature.parameters - and "num_experts" in signature.parameters - ): - kwargs["num_experts"] = kwargs.pop("global_num_experts") return select_experts(**kwargs) def recv_ffn_work_item( @@ -344,13 +336,8 @@ def recv_ffn_work_item( ) context = recv_output.context metadata = context.metadata - states = context.states - - token_nums_rankid_layeridx = ( - getattr(states, "token_nums_rankid_layeridx", None) - if states is not None - else None - ) + states = _require_async_transfer_state(context) + token_nums_rankid_layeridx = states.token_nums_rankid_layeridx if token_nums_rankid_layeridx is None: raise RuntimeError( "AFD async CAM FFN work item requires " @@ -359,36 +346,37 @@ def recv_ffn_work_item( total_num_tokens = max(1, int(token_nums_rankid_layeridx[0].item())) layer_idx = int(token_nums_rankid_layeridx[2].item()) - expert_token_nums_shared = ( - getattr(states, "expert_token_nums_shared", None) - if states is not None - else None - ) + expert_token_nums_shared = states.expert_token_nums_shared if expert_token_nums_shared is None: - shared_num_tokens = max(1, total_num_tokens) - else: - shared_num_tokens = max(1, int(expert_token_nums_shared[0].item())) + raise RuntimeError( + "AFD async CAM FFN work item requires " + "expert_token_nums_shared from async_dispatch_recv", + ) + shared_num_tokens = max(0, int(expert_token_nums_shared[0].item())) expert_token_nums = states.group_list - if isinstance(expert_token_nums, Tensor): - num_tokens = max(0, int(expert_token_nums.to(torch.int64).sum().item())) - else: - num_tokens = max(0, total_num_tokens - shared_num_tokens) + if expert_token_nums is None: + raise RuntimeError( + "AFD async CAM FFN work item requires expert_token_nums " + "from async_dispatch_recv", + ) + num_tokens = max( + 0, + int(expert_token_nums.to(torch.int64).sum().item()), + ) metadata.layer_idx = layer_idx metadata.stage_idx = stage_idx metadata.seq_lens = [num_tokens] - shared_slice_tokens = shared_num_tokens if shared_num_tokens > 0 else 100 - assert states, "payload should not have a None states" hidden_states = recv_output.hidden_states[:num_tokens] if states.expand_x_shared is not None: - states.expand_x_shared = states.expand_x_shared[:shared_slice_tokens] + states.expand_x_shared = states.expand_x_shared[:shared_num_tokens] if states.dynamic_scales is not None: states.dynamic_scales = states.dynamic_scales[:num_tokens] if states.dynamic_scales_shared is not None: states.dynamic_scales_shared = states.dynamic_scales_shared[ - :shared_slice_tokens + :shared_num_tokens ] return AFDAsyncFFNWorkItem( @@ -608,7 +596,7 @@ def recv_ffn_output( if topk_weights is None: topk_weights = pending_topk_weights - states = context.states + states = _require_async_transfer_state(context) _validate_topk_payload( topk_ids, topk_weights, @@ -761,7 +749,7 @@ def send_ffn_output( mandatory because CAM uses it to return results to Attention ranks. """ self._require_initialized() - states = context.states + states = _require_async_transfer_state(context) expand_x_shared = kwargs.get("expand_x_shared") if expand_x_shared is None: expand_x_shared = ffn_output @@ -815,6 +803,18 @@ def _require_initialized(self) -> None: raise RuntimeError("CAMAsyncAFDConnector is not initialized") +def _require_async_transfer_state( + context: AFDTransferContext, +) -> AFDAsyncTransferState: + states = context.states + if not isinstance(states, AFDAsyncTransferState): + raise RuntimeError( + "CAMAsyncAFDConnector requires AFDAsyncTransferState in the " + "transfer context", + ) + return states + + _CAM_LOG_SKIPPED_ARGS = frozenset({"comm_args", "comm_id", "group_name"}) _CAM_OP_IO_LOG_ENV = "AFD_CAM_OP_IO_LOG" diff --git a/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py b/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py index fae5a67c..b70d5cc9 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v2_async_cam_forward.py @@ -152,6 +152,12 @@ def run_attention_gate_afd_forward( llama_4_scaling, ) + # vLLM v0.26 profiles Attention with a local dummy forward. There is + # no matching FFN profile request, so launching CAM collectives here + # would send synthetic routing metadata into an unmatched data path. + if forward_context.in_profile_run: + continue + metadata = AFDTransferMetadata.create_attention_metadata( layer_idx=layer.layer_idx, stage_idx=stage_idx, diff --git a/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py b/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py index 17e4cda1..2f0ddbee 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py @@ -69,11 +69,11 @@ def compute_gate_topk( vllm_config.parallel_config.eplb_config.num_redundant_experts ) if mix_placement: - global_num_experts = ( + num_experts = ( config.n_shared_experts + config.n_routed_experts + num_redundant_experts ) else: - global_num_experts = config.n_routed_experts + num_redundant_experts + num_experts = config.n_routed_experts + num_redundant_experts routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) topk_weights, topk_ids = afd_connector.select_experts( hidden_states=hidden_states, @@ -89,7 +89,7 @@ def compute_gate_topk( mix_placement=mix_placement, num_logical_experts=router_logits.shape[1], num_shared_experts=config.n_shared_experts, - global_num_experts=global_num_experts, + num_experts=num_experts, ) if force_balanced_topk_ids_enabled(): topk_ids = _force_balanced_topk_ids( @@ -125,25 +125,37 @@ def compute_attention_gate_moe_ffn( quant_type = experts.quant_type if quant_type == QuantType.NONE: moe_weights = MoEWeights( - w1=experts.w13_weight, - w2=experts.w2_weight, - w1_bias=experts.w13_bias if experts.moe_config.has_bias else None, - w2_bias=experts.w2_bias if experts.moe_config.has_bias else None, + w1=experts.get_eplb_parameter("w13_weight"), + w2=experts.get_eplb_parameter("w2_weight"), + w1_bias=( + experts.get_eplb_parameter("w13_bias") + if experts.moe_config.has_bias + else None + ), + w2_bias=( + experts.get_eplb_parameter("w2_bias") + if experts.moe_config.has_bias + else None + ), ) elif quant_type == QuantType.W8A8: if experts.dynamic_eplb: moe_weights = MoEWeights( - w1=experts.w13_weight_list, - w2=experts.w2_weight_list, - w1_scale=experts.w13_weight_scale_fp32_list, - w2_scale=experts.w2_weight_scale_list, + w1=experts.get_eplb_parameter("w13_weight_list"), + w2=experts.get_eplb_parameter("w2_weight_list"), + w1_scale=experts.get_eplb_parameter( + "w13_weight_scale_fp32_list", + ), + w2_scale=experts.get_eplb_parameter("w2_weight_scale_list"), ) else: moe_weights = MoEWeights( - w1=[experts.w13_weight], - w2=[experts.w2_weight], - w1_scale=[experts.w13_weight_scale_fp32], - w2_scale=[experts.w2_weight_scale], + w1=[experts.get_eplb_parameter("w13_weight")], + w2=[experts.get_eplb_parameter("w2_weight")], + w1_scale=[ + experts.get_eplb_parameter("w13_weight_scale_fp32"), + ], + w2_scale=[experts.get_eplb_parameter("w2_weight_scale")], ) else: raise RuntimeError( @@ -174,7 +186,7 @@ def compute_attention_gate_moe_ffn( ) shared_output = experts._shared_experts(shared_input) - routed_output = unified_apply_mlp( + routed_output, _ = unified_apply_mlp( mlp_compute_input=MoEMlpComputeInput( hidden_states=hidden_states, group_list=group_list, diff --git a/afd_plugin/v1/worker/npu/attention_model_runner.py b/afd_plugin/v1/worker/npu/attention_model_runner.py index 3fd69e9c..9278b503 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner.py @@ -120,7 +120,8 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): afd_config=afd_config, ) self.afd_config = _with_dp_derived_afd_rank(vllm_config, self.afd_config) - rank, local_rank = _resolve_world_ranks() + rank, _ = _resolve_world_ranks() + local_rank = int(device.index) self.connector = AFDConnectorFactory.create_connector( rank, local_rank, @@ -828,7 +829,7 @@ def _dummy_run_inference_mode( self._afd_pending_metadata = None self._afd_async_moe_ubatch_metadata = None - # Upstream source: vLLM commit 68b0c3135, + # Upstream source: vLLM v0.26.0 commit 568afb3a1, # GPUModelRunner._warmup_and_capture. # Patch reason: AFD needs both single-stage and two-stage Ascend graph keys, # because live decode may fall below the DBO threshold. diff --git a/afd_plugin/v1/worker/npu/ffn_model_runner.py b/afd_plugin/v1/worker/npu/ffn_model_runner.py index c14f67d5..098afb2d 100644 --- a/afd_plugin/v1/worker/npu/ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/ffn_model_runner.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import torch from vllm.compilation.monitor import set_cudagraph_capturing_enabled @@ -32,6 +32,10 @@ AFDForwardContextMetadata, AFDTransferContext, ) +from afd_plugin.connectors.npu.async_cam import ( + AFDAsyncTransferState, + CAMAsyncAFDConnector, +) from afd_plugin.v1.worker.attention_model_runner import ( _resolve_world_ranks, _with_dp_derived_afd_rank, @@ -45,8 +49,9 @@ if TYPE_CHECKING: from vllm.sequence import IntermediateTensors - from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec + from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput from afd_plugin.connectors import AFDConnectorBase @@ -58,7 +63,7 @@ class AFDNPUFFNModelRunner(NPUModelRunner): afd_expected_role = "ffn" - def __init__(self, vllm_config: VllmConfig, device: object) -> None: + def __init__(self, vllm_config: VllmConfig, device: torch.device) -> None: afd_config = self.parse_config(vllm_config) super().__init__(vllm_config, device) @@ -68,7 +73,8 @@ def __init__(self, vllm_config: VllmConfig, device: object) -> None: afd_config=afd_config, ) self.afd_config = _with_dp_derived_afd_rank(vllm_config, self.afd_config) - rank, local_rank = _resolve_world_ranks() + rank, _ = _resolve_world_ranks() + local_rank = int(device.index) self.connector = AFDConnectorFactory.create_connector( rank, local_rank, @@ -82,6 +88,7 @@ def __init__(self, vllm_config: VllmConfig, device: object) -> None: current_platform.get_global_graph_pool() if self.use_aclgraph else None ) self.prof = create_afd_npu_profiler("ffn") + self._is_shutdown = False @staticmethod def parse_config(vllm_config: VllmConfig) -> AFDConfig: @@ -271,28 +278,25 @@ def _ffn_forward( ) return rank_ffn_output - def _ffn_forward_connector_driven(self) -> Any: + def _ffn_forward_connector_driven( + self, + ) -> torch.Tensor | AFDF2ATransferPayload | None: stage_idx = 0 rank_ffn_output = None - recv_work_item = getattr(self.connector, "recv_ffn_work_item", None) - send_work_item_output = getattr( - self.connector, - "send_ffn_work_item_output", - None, - ) - if not callable(recv_work_item) or not callable(send_work_item_output): - raise RuntimeError( - "connector-driven NPU FFN requires async connector work item APIs", - ) + connector = cast(CAMAsyncAFDConnector, self.connector) for _ in _ffn_layer_indices(self): - work_item = recv_work_item( + work_item = connector.recv_ffn_work_item( stage_idx=stage_idx, max_num_tokens=self.max_num_tokens, ) hidden_states = work_item.hidden_states metadata = work_item.context.metadata states = work_item.context.states + if not isinstance(states, AFDAsyncTransferState): + raise RuntimeError( + "CAM async FFN work item requires AFDAsyncTransferState", + ) layer_idx = work_item.layer_idx num_tokens = work_item.num_tokens afd_metadata = AFDForwardContextMetadata( @@ -323,7 +327,10 @@ def _ffn_forward_connector_driven(self) -> Any: expand_x_shared=states.expand_x_shared, dynamic_scales_shared=states.dynamic_scales_shared, ) - rank_ffn_output = send_work_item_output(work_item, rank_ffn_output) + rank_ffn_output = connector.send_ffn_work_item_output( + work_item, + rank_ffn_output, + ) return rank_ffn_output def capture_model( @@ -417,16 +424,20 @@ def _capture_graphs( } logger.debug("AFD NPU FFN captured ACL graph for key=%s", graph_key) - def sample_tokens(self, grammar_output: Any = None) -> Any: + def sample_tokens( + self, + grammar_output: GrammarOutput | None, + ) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors: raise RuntimeError("AFD NPU FFN runners do not sample tokens") def shutdown(self) -> None: + if self._is_shutdown: + return stop_afd_npu_profiler(self.prof) - self.connector.close() - try: - super().shutdown() - except AttributeError: - logger.debug("AFD NPU FFN parent model runner has no shutdown()") + if self.connector.is_initialized: + self.connector.close() + super().shutdown() + self._is_shutdown = True def _send_ffn_output( @@ -456,8 +467,7 @@ def _send_ffn_output( def _ffn_layer_indices(runner: AFDNPUFFNModelRunner) -> range | list[int]: num_layers = max(int(runner.num_layers or 0), 1) - afd_config = getattr(runner, "afd_config", None) - if afd_config is None or not bool(afd_config.compute_gate_on_attention): + if not runner.afd_config.compute_gate_on_attention: return range(num_layers) hf_config = runner.model_config.hf_config return [ diff --git a/afd_plugin/v1/worker/npu/ffn_worker.py b/afd_plugin/v1/worker/npu/ffn_worker.py index 9b89e409..e007be28 100644 --- a/afd_plugin/v1/worker/npu/ffn_worker.py +++ b/afd_plugin/v1/worker/npu/ffn_worker.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any import torch +from vllm.v1.worker.worker_base import CompilationTimes from vllm.v1.worker.workspace import init_workspace_manager from vllm_ascend.worker.worker import NPUWorker @@ -25,9 +26,12 @@ if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec + from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput logger = logging.getLogger(__name__) +FFN_SHUTDOWN_TIMEOUT_SECONDS = 5 + class AFDNPUFFNWorker(NPUWorker): """FFN worker that owns a connector-driven NPU daemon loop.""" @@ -72,10 +76,13 @@ def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None: self.model_runner.initialize_afd_connector() self.start_ffn_server_loop() - def compile_or_warm_up_model(self) -> float: - return 0.0 + def compile_or_warm_up_model(self) -> CompilationTimes: + return CompilationTimes(language_model=0.0, encoder=0.0) - def execute_model(self, scheduler_output: SchedulerOutput) -> None: + def execute_model( + self, + scheduler_output: SchedulerOutput, + ) -> ModelRunnerOutput | AsyncModelRunnerOutput | None: raise RuntimeError( "AFD NPU FFN workers are connector-driven; scheduler-driven " "execute_model() is not supported.", @@ -98,6 +105,13 @@ def ffn_worker_loop() -> None: try: self._run_ffn_server_loop() except Exception as exc: + shutdown_event = self._ffn_shutdown_event + if shutdown_event is not None and shutdown_event.is_set(): + logger.debug( + "AFD NPU FFN receive loop stopped during shutdown", + exc_info=True, + ) + return self._ffn_loop_error = exc logger.exception("AFD NPU FFN worker loop failed") @@ -142,17 +156,25 @@ def stop_ffn_server_loop(self) -> None: event = self._ffn_shutdown_event if event is not None: event.set() - try: - self.model_runner.shutdown() - finally: - thread = self._ffn_thread - if thread is not None: - thread.join(timeout=5) - self._ffn_thread = None - self._ffn_shutdown_event = None + + # CAM recv blocks in the connector operator. Release the communicator + # first so the daemon can observe the shutdown event, then wait for it + # before the parent runner releases model tensors. + self.model_runner.connector.close() + thread = self._ffn_thread + if thread is not None: + thread.join(timeout=FFN_SHUTDOWN_TIMEOUT_SECONDS) + if thread.is_alive(): + raise RuntimeError( + "AFD NPU FFN worker loop did not stop after connector close", + ) + self._ffn_thread = None + self._ffn_shutdown_event = None self.raise_ffn_loop_error_if_any() def shutdown(self) -> None: + # Stop the connector-driven daemon before NPUWorker releases the model + # runner and its tensors. self.stop_ffn_server_loop() super().shutdown() diff --git a/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py b/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py index 16bb8c7a..13a370be 100644 --- a/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py +++ b/afd_plugin/v1/worker/npu/npu_ubatch_wrapper.py @@ -48,7 +48,7 @@ def _cat_ubatch_outputs( ) -> AscendLastRankOutput: """Preserve the current Ascend model-output structure across ubatches. - Upstream source: vLLM commit 68b0c3135, + Upstream source: vLLM v0.26.0 commit 568afb3a1, ``gpu_ubatch_wrapper._cat_ubatch_outputs``. Ascend auxiliary hidden states use ``tuple[Tensor, list[Tensor]]`` rather than upstream's tuple of tensors, so this plugin-owned wrapper concatenates that concrete nested contract. diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index f1d5523b..7d637063 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -204,13 +204,21 @@ current async MoE metadata path does not support it. ## Requirements -The historical checked-in recipe was verified with: +The CAM async v0.26 path was verified with: - Ascend 910C; -- `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler` (legacy only); +- CANN 9.0.1; +- runtime image build `nightly-main-a3-openeuler-20260801230444_aarch64`; +- vLLM v0.26.0 at commit `568afb3a1`; +- vLLM-Ascend branch `releases/v0.26.0rc` at commit `80d8c194f`; - the included `CAM_ascend910_93_openEuler_aarch64.run` installer; - `umdk_cam_op_lib-208.1.0b1-cp311-cp311-linux_aarch64.whl`. +The nightly image identifier records the validation environment; it is not a +promise of a stable public pull tag. Some development package metadata in that +image still reports a `0.19.1rc2.dev1327` version. The source commits above are +the authoritative compatibility baseline for this v0.26 upgrade. + Install the CAM packages from the repository root inside the container: ```bash @@ -223,7 +231,9 @@ the Ascend plugin enabled. The complete recipe includes all tuning variables; the essential setup is: ```bash -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} +export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md index 59c93db9..3121d589 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md @@ -32,11 +32,15 @@ AFD provides the following backend-specific connectors: ## Image and Hardware Requirements - Hardware: Ascend 910C only. -- Image: `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler`. +- CANN: 9.0.1. +- Validated runtime image build: + `nightly-main-a3-openeuler-20260801230444_aarch64`. +- vLLM: v0.26.0 at commit `568afb3a1`. +- vLLM-Ascend: branch `releases/v0.26.0rc` at commit `80d8c194f`. -```bash -docker pull quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler -``` +The nightly build name identifies the validation environment and is not +documented as a stable public image tag. Provision an equivalent runtime at +the commits above rather than using the former v0.19.1rc1 image command. ## Installing Operator Packages @@ -264,7 +268,9 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} +export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -329,7 +335,9 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} +export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -395,7 +403,9 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} +export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True diff --git a/tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py b/tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py index b0d8fae2..e8da9e6c 100644 --- a/tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py +++ b/tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """Opt-in NPU E2E smoke test for DeepSeekV2-Lite async CAM. -Skipped unless AFD_NPU_E2E_MODEL is set to a local DeepSeekV2-Lite model path. +The smoke limits vLLM memory utilization so the CAM HCCL buffer retains +device-memory headroom. Skipped unless ``AFD_NPU_ASYNC_CAM_E2E_MODEL`` (or the +shared ``AFD_NPU_E2E_MODEL`` fallback) points to a local model path. The smoke topology uses 4 NPUs: Attention: DP=1, TP=2 @@ -11,6 +13,7 @@ from __future__ import annotations +import json import os import subprocess import sys @@ -22,12 +25,17 @@ REPO_ROOT = Path(__file__).resolve().parents[4] RUNNER = REPO_ROOT / "tests" / "e2e" / "runner.py" -ASYNC_CAM_EXTRA_CONFIG = ( - '{"dynamicQuant":0,"async_moe_ubatching":false,' - '"async_moe_num_ubatches":2,"async_moe_split":"request",' - '"attn_ranks_per_dp":2}' +CAM_VENDOR_PATH = Path( + "/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM", ) -DSV2_ACCOUNTING_PROMPT = "\n".join( +CAM_OP_API_PATH = CAM_VENDOR_PATH / "op_api" +CAM_OP_API_LIB_PATH = CAM_OP_API_PATH / "lib" +CAM_ATTENTION_RANKS = 2 +CAM_FFN_RANKS = 2 +CAM_REQUIRED_NPUS = CAM_ATTENTION_RANKS + CAM_FFN_RANKS +CAM_HCCL_BUFFSIZE = "4096" +CAM_GPU_MEMORY_UTILIZATION = "0.75" +CAM_ACCOUNTING_PROMPT = "\n".join( [ "<|im_start|>system", "You are a professional accountant. Answer questions using accounting " @@ -57,35 +65,51 @@ def _npu_list() -> list[str]: return [ item.strip() - for item in os.environ.get("AFD_NPU_ASYNC_CAM_E2E_DEVICES", "0,1,2,3").split( - ",", - ) + for item in os.environ.get( + "AFD_NPU_ASYNC_CAM_E2E_DEVICES", + "0,1,2,3", + ).split(",") if item.strip() ] def _model_path() -> str: - model = os.environ.get("AFD_NPU_E2E_MODEL") + model = os.environ.get("AFD_NPU_ASYNC_CAM_E2E_MODEL") or os.environ.get( + "AFD_NPU_E2E_MODEL", + ) if not model: - pytest.skip("set AFD_NPU_E2E_MODEL to run async CAM NPU E2E tests") + pytest.skip("set AFD_NPU_ASYNC_CAM_E2E_MODEL to run async CAM NPU E2E tests") return model +def _prepend_env_paths( + env: dict[str, str], + name: str, + *paths: Path, +) -> None: + existing_paths = [path for path in env.get(name, "").split(os.pathsep) if path] + ordered_paths = [str(path) for path in paths] + env[name] = os.pathsep.join(dict.fromkeys([*ordered_paths, *existing_paths])) + + def _async_cam_env() -> dict[str, str]: env = os.environ.copy() env.setdefault("VLLM_USE_V1", "1") - env.setdefault("HCCL_BUFFSIZE", "6144") + env["HCCL_BUFFSIZE"] = CAM_HCCL_BUFFSIZE env.setdefault("ASCEND_LAUNCH_BLOCKING", "1") env.setdefault("VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL", "1") env.setdefault("VLLM_ASCEND_ENABLE_FLASHCOMM1", "1") - cam_op_lib = Path("/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib") - if cam_op_lib.exists(): - current_ld_library_path = env.get("LD_LIBRARY_PATH", "") - env["LD_LIBRARY_PATH"] = ( - str(cam_op_lib) - if not current_ld_library_path - else f"{cam_op_lib}:{current_ld_library_path}" - ) + _prepend_env_paths( + env, + "LD_LIBRARY_PATH", + CAM_OP_API_PATH, + CAM_OP_API_LIB_PATH, + ) + _prepend_env_paths( + env, + "ASCEND_CUSTOM_OPP_PATH", + CAM_VENDOR_PATH, + ) python_paths = [ str(path) for path in ( @@ -104,19 +128,63 @@ def _async_cam_env() -> dict[str, str]: return env +def _async_cam_extra_config(model_path: str) -> str: + dynamic_quant = int( + os.environ.get( + "AFD_NPU_ASYNC_CAM_E2E_DYNAMIC_QUANT", + ( + "1" + if (Path(model_path) / "quant_model_description.json").is_file() + else "0" + ), + ), + ) + return json.dumps( + { + "dynamicQuant": dynamic_quant, + "async_moe_ubatching": False, + "async_moe_num_ubatches": 2, + "async_moe_split": "request", + "attn_ranks_per_dp": 2, + }, + separators=(",", ":"), + ) + + +def _async_cam_common_vllm_args() -> list[str]: + return [ + "--trust-remote-code", + "--max-num-seqs", + "8", + "--max-num-batched-tokens", + "8000", + "--gpu-memory-utilization", + CAM_GPU_MEMORY_UTILIZATION, + "--no-enable-prefix-caching", + ] + + @pytest.mark.npu @pytest.mark.e2e @pytest.mark.slow def test_deepseek_v2_lite_async_cam_attn_dp1tp2_ffn_dp2ep2_smoke(): npus = _npu_list() - if len(npus) < 4: - pytest.skip(f"async CAM smoke requires 4 NPUs; got {len(npus)}") + if len(npus) < CAM_REQUIRED_NPUS: + pytest.skip( + f"async CAM smoke requires {CAM_REQUIRED_NPUS} NPUs; got {len(npus)}", + ) + model_path = _model_path() + common_vllm_args = [ + f"--common-vllm-arg={argument}" for argument in _async_cam_common_vllm_args() + ] command = [ sys.executable, str(RUNNER), "--model", - _model_path(), + model_path, + "--served-model-name-prefix", + "cam-async", "--vllm-bin", os.environ.get("AFD_NPU_E2E_VLLM_BIN", "vllm"), "--device-backend", @@ -126,19 +194,19 @@ def test_deepseek_v2_lite_async_cam_attn_dp1tp2_ffn_dp2ep2_smoke(): "--afd-async", "--compute-gate-on-attention", "--afd-connector-extra-config", - ASYNC_CAM_EXTRA_CONFIG, + _async_cam_extra_config(model_path), "--num-attention-ranks", - "2", + str(CAM_ATTENTION_RANKS), "--num-ffn-ranks", - "2", + str(CAM_FFN_RANKS), "--attention-tp-size", "2", "--ffn-tp-size", "1", "--attention-gpus", - ",".join(npus[:2]), + ",".join(npus[:CAM_ATTENTION_RANKS]), "--ffn-gpus", - ",".join(npus[2:4]), + ",".join(npus[CAM_ATTENTION_RANKS:CAM_REQUIRED_NPUS]), "--api-port-base", os.environ.get("AFD_NPU_ASYNC_CAM_E2E_API_PORT", "19080"), "--afd-port", @@ -146,7 +214,7 @@ def test_deepseek_v2_lite_async_cam_attn_dp1tp2_ffn_dp2ep2_smoke(): "--startup-timeout", os.environ.get("AFD_NPU_E2E_STARTUP_TIMEOUT", "900"), "--prompt", - DSV2_ACCOUNTING_PROMPT, + CAM_ACCOUNTING_PROMPT, "--max-tokens", os.environ.get("AFD_NPU_ASYNC_CAM_E2E_MAX_TOKENS", "32"), "--temperature", @@ -155,12 +223,7 @@ def test_deepseek_v2_lite_async_cam_attn_dp1tp2_ffn_dp2ep2_smoke(): "1", "--request-concurrency", "1", - "--common-vllm-arg=--trust-remote-code", - "--common-vllm-arg=--max-num-seqs", - "--common-vllm-arg=8", - "--common-vllm-arg=--max-num-batched-tokens", - "--common-vllm-arg=8000", - "--common-vllm-arg=--no-enable-prefix-caching", + *common_vllm_args, ] max_model_len = os.environ.get("AFD_NPU_ASYNC_CAM_E2E_MAX_MODEL_LEN") diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index ee86a51e..a73cb52b 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -103,7 +103,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--model", required=True, - help="DeepSeekV2-Lite model path or Hugging Face model id.", + help="Model path or Hugging Face model id.", ) parser.add_argument( "--vllm-bin", diff --git a/tests/e2e/test_runner.py b/tests/e2e/test_runner.py index f6f66ee2..dbbd9d42 100644 --- a/tests/e2e/test_runner.py +++ b/tests/e2e/test_runner.py @@ -8,6 +8,7 @@ import pytest from tests.e2e import runner +from tests.e2e.models.deepseek_v2_lite import test_async_cam_npu as async_cam_e2e from tests.e2e.runner import build_vllm_command @@ -155,6 +156,39 @@ def test_runner_drops_flashcomm_for_npu_role_without_tp(monkeypatch): assert "VLLM_ASCEND_ENABLE_FLASHCOMM1" not in env +def test_async_cam_env_registers_cam_vendor_before_existing_paths(monkeypatch): + existing_opp = "/opt/existing/opp/vendor" + existing_lib = "/opt/existing/lib" + monkeypatch.setenv( + "ASCEND_CUSTOM_OPP_PATH", + f"{existing_opp}:{async_cam_e2e.CAM_VENDOR_PATH}", + ) + monkeypatch.setenv( + "LD_LIBRARY_PATH", + ":".join( + [ + existing_lib, + str(async_cam_e2e.CAM_OP_API_LIB_PATH), + str(async_cam_e2e.CAM_OP_API_PATH), + ], + ), + ) + monkeypatch.setenv("HCCL_BUFFSIZE", "8192") + + env = async_cam_e2e._async_cam_env() + + assert env["ASCEND_CUSTOM_OPP_PATH"].split(":") == [ + str(async_cam_e2e.CAM_VENDOR_PATH), + existing_opp, + ] + assert env["LD_LIBRARY_PATH"].split(":") == [ + str(async_cam_e2e.CAM_OP_API_PATH), + str(async_cam_e2e.CAM_OP_API_LIB_PATH), + existing_lib, + ] + assert env["HCCL_BUFFSIZE"] == async_cam_e2e.CAM_HCCL_BUFFSIZE + + def test_runner_fails_fast_when_server_exits_before_api_is_ready(): args = _args() process = argparse.Namespace( diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index e9db4a79..45c65156 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -10,6 +10,8 @@ pytest.importorskip("torch") pytest.importorskip("torch_npu") +import torch # noqa: E402 + from afd_plugin.connectors import ( # noqa: E402 AFDA2FTransferPayload, AFDConnectorFactory, @@ -446,7 +448,7 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): _FakeScalar(11), ], expert_token_nums_shared=[_FakeScalar(2)], - group_list="groups", + group_list=torch.tensor([2, 3], dtype=torch.int64), dynamic_scales=_FakeTensorLike("scales"), expand_x_shared=_FakeTensorLike("shared-hidden"), dynamic_scales_shared=_FakeTensorLike("shared-scales"), @@ -484,9 +486,8 @@ def test_async_ffn_work_item_uses_expert_counts_for_routed_tokens(monkeypatch): import torch - # The routed token count comes from summing the per-expert group_list, which - # the connector only does when it is a real tensor (a list falls back to - # total - shared). The counts below sum to 6. + # The routed token count comes from summing the per-expert group_list. + # The counts below sum to 6. group_list = torch.tensor( [1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int64, @@ -509,7 +510,7 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): _FakeScalar(0), _FakeScalar(23), ], - expert_token_nums_shared=[_FakeScalar(1)], + expert_token_nums_shared=[_FakeScalar(0)], group_list=group_list, expand_x_shared=_FakeTensorLike("shared-hidden"), dynamic_scales_shared=_FakeTensorLike("shared-scales"), @@ -526,12 +527,12 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): states = work_item.context.states assert work_item.layer_idx == 23 assert work_item.total_num_tokens == 6 - assert work_item.shared_num_tokens == 1 + assert work_item.shared_num_tokens == 0 assert work_item.num_tokens == 6 assert work_item.hidden_states == "hidden[:6]" assert work_item.context.metadata.seq_lens == [6] - assert states.expand_x_shared == "shared-hidden[:1]" - assert states.dynamic_scales_shared == "shared-scales[:1]" + assert states.expand_x_shared == "shared-hidden[:0]" + assert states.dynamic_scales_shared == "shared-scales[:0]" def test_async_send_ffn_work_item_output_preserves_all_shared_passthrough( @@ -569,6 +570,7 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): _FakeScalar(7), ], expert_token_nums_shared=[_FakeScalar(5)], + group_list=torch.zeros(8, dtype=torch.int64), expand_x_shared=_FakeTensorLike("shared-hidden"), dynamic_scales_shared=_FakeTensorLike("shared-scales"), ) @@ -608,7 +610,7 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): ] -def test_async_select_experts_maps_legacy_global_num_experts(monkeypatch): +def test_async_select_experts_uses_v026_num_experts_contract(monkeypatch): calls = [] fake_package = ModuleType("vllm_ascend") fake_ops = ModuleType("vllm_ascend.ops") @@ -635,7 +637,7 @@ def select_experts(*, num_experts=-1, **kwargs): _afd_config(role="attention"), ) - result = connector.select_experts(router_logits="logits", global_num_experts=8) + result = connector.select_experts(router_logits="logits", num_experts=8) assert result == ("weights", "ids") assert calls == [(8, {"router_logits": "logits"})] diff --git a/tests/unit/model_executor/models/test_forward_context.py b/tests/unit/model_executor/models/test_forward_context.py index b863470c..068e12da 100644 --- a/tests/unit/model_executor/models/test_forward_context.py +++ b/tests/unit/model_executor/models/test_forward_context.py @@ -249,6 +249,63 @@ def run_norm(hidden_states, residual): assert torch.equal(output["residual"], scheduled_residual) +def test_async_cam_profile_forward_skips_real_connector_io(monkeypatch): + from afd_plugin.model_executor.models.npu import ( + deepseek_v2_async_cam_forward as async_forward, + ) + + forward_context = SimpleNamespace( + in_profile_run=True, + ubatch_idx=0, + ) + monkeypatch.setattr(async_forward, "get_forward_context", lambda: forward_context) + + connector_calls = [] + connector = SimpleNamespace( + send_attn_output=lambda *args, **kwargs: connector_calls.append("send"), + recv_ffn_output=lambda *args, **kwargs: connector_calls.append("recv"), + ) + afd_metadata = SimpleNamespace(connector=connector, stage_idx=0) + + class _ProfileMoELayer: + is_moe_layer = True + layer_idx = 0 + + def compute_attn_output( + self, + positions, + hidden_states, + residual, + llama_4_scaling, + ): + return ( + hidden_states + 1, + residual, + torch.ones((hidden_states.shape[0], 1)), + torch.zeros((hidden_states.shape[0], 1), dtype=torch.int32), + torch.ones((hidden_states.shape[0], 1)), + ) + + model = SimpleNamespace( + layers=[_ProfileMoELayer(), _ProfileMoELayer()], + start_layer=0, + end_layer=2, + ) + hidden_states = torch.zeros((2, 4)) + + output, residual = async_forward.run_attention_gate_afd_forward( + model, + hidden_states, + None, + torch.arange(2), + afd_metadata, + ) + + assert torch.equal(output, hidden_states + 2) + assert residual is None + assert connector_calls == [] + + def test_deepseek_afd_wrapper_keeps_full_model_compile_enabled(): source = Path("afd_plugin/model_executor/models/deepseek_v2.py").read_text() @@ -457,7 +514,12 @@ def test_deepseek_afd_ffn_path_reuses_ascend_moe_mlp_after_attention_gate(): assert "AFDF2ATransferPayload(" in compute_moe assert "MoEMlpComputeInput(" in compute_moe assert "unified_apply_mlp(" in compute_moe + assert "routed_output, _ = unified_apply_mlp(" in compute_moe assert "quant_type == QuantType.W8A8" in compute_moe + assert 'experts.get_eplb_parameter("w13_weight")' in compute_moe + assert 'experts.get_eplb_parameter("w2_weight")' in compute_moe + assert "experts.w13_weight" not in compute_moe + assert "experts.w2_weight" not in compute_moe assert "w13_weight_scale_fp32" in compute_moe assert "w13_weight_scale_fp32_list" in compute_moe assert "w2_weight_scale_list" in compute_moe diff --git a/tests/unit/v1/worker/test_npu_device_contract.py b/tests/unit/v1/worker/test_npu_device_contract.py new file mode 100644 index 00000000..ff673315 --- /dev/null +++ b/tests/unit/v1/worker/test_npu_device_contract.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project + +from pathlib import Path + + +def test_npu_connectors_use_model_device_index_for_dp_workers(): + for source_path in ( + Path("afd_plugin/v1/worker/npu/attention_model_runner.py"), + Path("afd_plugin/v1/worker/npu/ffn_model_runner.py"), + ): + source = source_path.read_text() + + assert "rank, _ = _resolve_world_ranks()" in source + assert "local_rank = int(device.index)" in source + assert "rank, local_rank = _resolve_world_ranks()" not in source diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index 41e106cf..c59976f4 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -346,6 +346,7 @@ def _new_ffn_runner(): runner = object.__new__(AFDNPUFFNModelRunner) runner.prof = None runner.device = SimpleNamespace(type="npu") + runner._is_shutdown = False return runner @@ -1142,6 +1143,37 @@ def test_npu_ffn_runner_requires_compute_hook(monkeypatch): runner.execute_ffn_step(dp_metadata_list={0: _FakeDPMetadata([1])}) +def test_npu_ffn_runner_shutdown_is_idempotent(monkeypatch): + _require_npu_runtime() + from afd_plugin.v1.worker.npu import ffn_model_runner + + runner = _new_ffn_runner() + close_calls = [] + profiler_calls = [] + parent_calls = [] + runner.connector = SimpleNamespace( + is_initialized=True, + close=lambda: close_calls.append(True), + ) + monkeypatch.setattr( + ffn_model_runner, + "stop_afd_npu_profiler", + lambda profiler: profiler_calls.append(profiler), + ) + monkeypatch.setattr( + ffn_model_runner.NPUModelRunner, + "shutdown", + lambda self: parent_calls.append(self), + ) + + runner.shutdown() + runner.shutdown() + + assert close_calls == [True] + assert profiler_calls == [None] + assert parent_calls == [runner] + + def test_npu_ffn_worker_scheduler_execute_model_fails_fast(): worker = _new_ffn_worker() @@ -1149,6 +1181,15 @@ def test_npu_ffn_worker_scheduler_execute_model_fails_fast(): worker.execute_model(scheduler_output=object()) +def test_npu_ffn_worker_reports_zero_compilation_times(): + worker = _new_ffn_worker() + + compilation_times = worker.compile_or_warm_up_model() + + assert compilation_times.language_model == 0.0 + assert compilation_times.encoder == 0.0 + + def test_npu_ffn_worker_loop_error_is_propagated(caplog): worker = _new_ffn_worker() worker._ffn_thread = None @@ -1180,6 +1221,117 @@ def fail_loop(): assert "AFD NPU FFN worker loop failed" in caplog.text +def test_npu_ffn_worker_ignores_receive_error_during_shutdown(caplog): + worker = _new_ffn_worker() + worker._ffn_thread = None + worker._ffn_shutdown_event = None + worker._ffn_loop_error = None + worker.model_runner = SimpleNamespace( + connector=SimpleNamespace(is_initialized=True), + ) + + def stop_while_receiving(): + worker._ffn_shutdown_event.set() + raise RuntimeError("CAM recv interrupted by connector close") + + worker._run_ffn_server_loop = stop_while_receiving + + with caplog.at_level( + logging.ERROR, + logger="afd_plugin.v1.worker.npu.ffn_worker", + ): + worker.start_ffn_server_loop() + assert worker._ffn_thread is not None + worker._ffn_thread.join(timeout=5) + + worker.raise_ffn_loop_error_if_any() + assert "AFD NPU FFN worker loop failed" not in caplog.text + + +def test_npu_ffn_worker_stops_loop_before_parent_shutdown(monkeypatch): + _require_npu_runtime() + from afd_plugin.v1.worker.npu import ffn_worker + + worker = _new_ffn_worker() + worker._ffn_shutdown_event = threading.Event() + calls = [] + + class _StoppingThread: + def join(self, timeout): + calls.append(("join", timeout)) + + def is_alive(self): + return False + + connector = SimpleNamespace(close=lambda: calls.append(("close", None))) + worker._ffn_thread = _StoppingThread() + worker.model_runner = SimpleNamespace(connector=connector) + monkeypatch.setattr( + ffn_worker.NPUWorker, + "shutdown", + lambda self: calls.append(("parent", None)), + ) + + worker.shutdown() + + assert calls == [ + ("close", None), + ("join", ffn_worker.FFN_SHUTDOWN_TIMEOUT_SECONDS), + ("parent", None), + ] + + +def test_npu_ffn_worker_preserves_live_thread_after_shutdown_timeout(monkeypatch): + _require_npu_runtime() + from afd_plugin.v1.worker.npu import ffn_worker + + worker = _new_ffn_worker() + shutdown_event = threading.Event() + worker._ffn_shutdown_event = shutdown_event + calls = [] + + class _StoppingThread: + alive = True + + def join(self, timeout): + calls.append(("join", timeout)) + + def is_alive(self): + return self.alive + + thread = _StoppingThread() + worker._ffn_thread = thread + worker.model_runner = SimpleNamespace( + connector=SimpleNamespace(close=lambda: calls.append(("close", None))), + ) + monkeypatch.setattr( + ffn_worker.NPUWorker, + "shutdown", + lambda self: calls.append(("parent", None)), + ) + + with pytest.raises(RuntimeError, match="did not stop"): + worker.shutdown() + + assert worker._ffn_thread is thread + assert worker._ffn_shutdown_event is shutdown_event + assert shutdown_event.is_set() + assert ("parent", None) not in calls + + thread.alive = False + worker.shutdown() + + assert worker._ffn_thread is None + assert worker._ffn_shutdown_event is None + assert calls == [ + ("close", None), + ("join", ffn_worker.FFN_SHUTDOWN_TIMEOUT_SECONDS), + ("close", None), + ("join", ffn_worker.FFN_SHUTDOWN_TIMEOUT_SECONDS), + ("parent", None), + ] + + def test_npu_ffn_worker_uses_connector_driven_loop_for_async_connector(): worker = _new_ffn_worker() event = threading.Event() From f0a34e68e3d0f7c054645371ca30694841e30bc2 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Wed, 5 Aug 2026 12:35:40 +0800 Subject: [PATCH 08/10] Address v0.26 review documentation fixes Signed-off-by: jiangkuaixue123 --- .gitignore | 2 ++ afd_plugin/compat/patches/npu/ascend_platform.py | 15 ++++++++------- afd_plugin/model_executor/models/deepseek_v2.py | 2 +- afd_plugin/v1/worker/ubatch_wrapper.py | 6 ++---- docs/design/module/compatibility_and_patches.md | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 830f1676..661f16c8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ sdist/ var/ wheels/ share/python-wheels/ +# Project-local metadata generated by editable installs. +vllm_afd_plugin.egg-info/ *.egg-info/ .installed.cfg *.egg diff --git a/afd_plugin/compat/patches/npu/ascend_platform.py b/afd_plugin/compat/patches/npu/ascend_platform.py index 66219994..13e0140a 100644 --- a/afd_plugin/compat/patches/npu/ascend_platform.py +++ b/afd_plugin/compat/patches/npu/ascend_platform.py @@ -28,13 +28,14 @@ class _AFDDBOConfigSnapshot: def apply_afd_ascend_dbo_config_patch() -> bool: """Preserve AFD-owned DBO settings during vLLM-Ascend config normalization. - vLLM-Ascend's platform compatibility pass disables DBO/ubatching fields for - ordinary NPU runs. AFD owns its NPU ubatching path, so this patch snapshots - those fields for AFD-enabled configs, lets upstream normalization run, then - restores the AFD DBO values. The patch is a no-op when vLLM-Ascend is not - importable. Returns whether this process has installed the wrapper (or had - already installed it), so callers do not cache a failed early import during - plugin initialization. + vLLM-Ascend's platform compatibility pass disables DBO/ubatching fields and + can rewrite ``all2all_backend`` for ordinary NPU runs. AFD owns its NPU + ubatching path, so this patch snapshots those fields for AFD-enabled configs, + lets upstream normalization run, then restores the AFD DBO values and + backend. The patch is a no-op when vLLM-Ascend is not importable. Returns + whether this process has installed the wrapper (or had already installed + it), so callers do not cache a failed early import during plugin + initialization. """ try: diff --git a/afd_plugin/model_executor/models/deepseek_v2.py b/afd_plugin/model_executor/models/deepseek_v2.py index aa302b37..90ee2462 100644 --- a/afd_plugin/model_executor/models/deepseek_v2.py +++ b/afd_plugin/model_executor/models/deepseek_v2.py @@ -241,7 +241,7 @@ class AFDDeepseekV2RemoteExpertsMoE(native.DeepseekV2MoE): # Patch reason: native DeepseekV2MoE constructs local routed/shared experts. # Patch functionality: preserve the native MoE forward contract while # constructing only the gate owned by Attention and a parameter-free proxy. - # Signature: AFD-owned; adds layer_idx and gate_placement and omits + # Signature: AFD-owned; adds layer_idx and compute_gate_on_attention and omits # quant_config because no local expert kernel is constructed. # Upstream: vLLM v0.26.0, vllm/model_executor/models/deepseek_v2.py # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 diff --git a/afd_plugin/v1/worker/ubatch_wrapper.py b/afd_plugin/v1/worker/ubatch_wrapper.py index 6ce39e13..d0d97681 100644 --- a/afd_plugin/v1/worker/ubatch_wrapper.py +++ b/afd_plugin/v1/worker/ubatch_wrapper.py @@ -8,7 +8,7 @@ from __future__ import annotations from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext +from contextlib import nullcontext from typing import Any import torch @@ -44,9 +44,7 @@ def configure_afd_context_provider(self, provider: Any) -> None: # Upstream: vLLM v0.26.0, vllm/v1/worker/gpu_ubatch_wrapper.py # Commit: 568afb3a13806beb53bb2e6bd518269357b237c0 @staticmethod - def _create_sm_control_context( - vllm_config: VllmConfig, - ) -> AbstractContextManager[None]: + def _create_sm_control_context(vllm_config: VllmConfig): # ### PATCH START: leave all SMs visible to AFD compute and communication. if is_afd_active(vllm_config): return nullcontext() diff --git a/docs/design/module/compatibility_and_patches.md b/docs/design/module/compatibility_and_patches.md index 91b487f3..cbdcb2e0 100644 --- a/docs/design/module/compatibility_and_patches.md +++ b/docs/design/module/compatibility_and_patches.md @@ -109,7 +109,7 @@ not the package dependency policy. | [`config_validation.py`](../../../afd_plugin/compat/patches/config_validation.py): `EngineArgs.create_engine_config`, `VllmConfig.__post_init__` | For AFD-owned ubatching with a non-DeepEP backend, temporarily presents `deepep_low_latency` during upstream validation and restores the configured backend. After upstream platform normalization, maps an initial `worker_cls="auto"` to the role-specific CUDA or standard Ascend AFD worker. | Imported by `register_afd`; accepts the target version, development versions, or missing version metadata. Saves originals on upstream modules under AFD-specific attributes before installing wrappers. Explicit worker paths and non-AFD configs are not remapped. | [`test_config_validation.py`](../../../tests/unit/compat/patches/test_config_validation.py) covers backend relaxation, four role/platform mappings, explicit and non-AFD preservation, repeated validation, unsupported platforms, and dev versions. | Remove the backend branch when upstream validation distinguishes plugin-owned ubatching; remove worker mapping when vLLM offers plugin-owned role-aware worker selection. | | [`engine_core.py`](../../../afd_plugin/compat/patches/engine_core.py): `EngineCore.__init__`, `_initialize_kv_caches`, `shutdown`; `EngineCoreProc.run_busy_loop`; `DPEngineCoreProc.run_busy_loop` | AFD FFN becomes a connector daemon: construct executor, skip scheduler/KV setup, return an empty KV-shaped result on late paths, start/monitor/stop the FFN worker loop, and use FFN-safe shutdown. Non-FFN branches copy pinned upstream behavior. | Imported by `register_afd`; **no patch-local version guard and no saved-original sentinel**. Direct class assignment means the package pin and review discipline are the compatibility guard. | [`test_engine_core.py`](../../../tests/unit/compat/patches/test_engine_core.py) covers FFN initialization, non-FFN behavior, and daemon start/stop; role runtime tests cover error propagation. | Remove when vLLM offers a headless connector-daemon engine lifecycle or an executor mode that does not require scheduler/KV ownership. | | [`npu/ascend_platform.py`](../../../afd_plugin/compat/patches/npu/ascend_platform.py): `NPUPlatform.check_and_update_config` | Snapshots AFD DBO state, runs upstream normalization, and restores configured `enable_dbo`, `ubatch_size`, and `all2all_backend` in `finally`; non-AFD behavior is unchanged. | Called through `apply_afd_ascend_patches_if_needed`; no version guard. Saves the original on the class and uses a class sentinel. The runtime facade caches success only after the wrapper is installed, so an early missing vLLM-Ascend import remains retryable. | [`test_runtime.py`](../../../tests/unit/compat/test_runtime.py) and [`test_npu_runtime.py`](../../../tests/unit/v1/worker/test_npu_runtime.py). | Remove when vLLM-Ascend recognizes plugin-owned DBO workers or no longer clears these fields. | -| [`npu/force_load_balance.py`](../../../afd_plugin/compat/patches/npu/force_load_balance.py): `AscendFusedMoE.__init__`, `AscendW8A8DynamicFusedMoEMethod.apply` | Adds AFD profiling configuration and replaces routed expert IDs with a deterministic balanced buffer only when the layer-owned switch is enabled; normal model-selected routing remains unchanged. This switch changes outputs and is not a correctness feature. | Imported only when vLLM-Ascend is discoverable; **no patch-local version guard or explicit reload sentinel**. Functions copy the current upstream bodies with marked AFD deltas. | [`test_force_load_balance.py`](../../../tests/unit/compat/patches/test_force_load_balance.py) covers buffer bounds, determinism, growth, override, and pass-through. | Upstream a deterministic expert-routing profiling hook in vLLM-Ascend, then delete both copied functions. | +| [`npu/force_load_balance.py`](../../../afd_plugin/compat/patches/npu/force_load_balance.py): `AscendW8A8DynamicFusedMoEMethod.__init__`, `AscendW8A8DynamicFusedMoEMethod.apply` | Captures AFD profiling configuration as method-owned state and replaces routed expert IDs with a deterministic balanced buffer only when the method-owned switch is enabled; normal model-selected routing remains unchanged. This switch changes outputs and is not a correctness feature. | Imported only when vLLM-Ascend is discoverable; **no patch-local version guard or explicit reload sentinel**. Functions copy the current upstream bodies with marked AFD deltas. | [`test_force_load_balance.py`](../../../tests/unit/compat/patches/test_force_load_balance.py) covers buffer bounds, determinism, growth, override, and pass-through. | Upstream a deterministic expert-routing profiling hook in vLLM-Ascend, then delete both copied functions. | ## Non-patch compatibility adapters From 08e851aefdfd1d3abc3ec9a73b92a6c766887933 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Wed, 5 Aug 2026 16:05:58 +0800 Subject: [PATCH 09/10] Delay NPU runtime patches until worker startup Signed-off-by: jiangkuaixue123 --- afd_plugin/__init__.py | 13 ++-------- afd_plugin/compat/npu/__init__.py | 2 ++ afd_plugin/compat/npu/runtime.py | 24 ++++++++++++------- .../compat/patches/config_validation.py | 6 +++-- afd_plugin/v1/worker/npu/ffn_worker.py | 5 ++++ .../compat/patches/test_config_validation.py | 6 ++--- 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/afd_plugin/__init__.py b/afd_plugin/__init__.py index 81b038a5..f53a4665 100644 --- a/afd_plugin/__init__.py +++ b/afd_plugin/__init__.py @@ -113,17 +113,8 @@ def register_afd() -> None: exc_info=True, ) - try: - from afd_plugin.compat.npu import apply_afd_ascend_patches_if_needed - - apply_afd_ascend_patches_if_needed() - if importlib.util.find_spec("vllm_ascend") is not None: - import afd_plugin.compat.patches.npu.force_load_balance # noqa: F401 - except Exception: - _logger.debug( - "AFD plugin: Ascend compatibility patches could not be applied", - exc_info=True, - ) + # NPU compatibility patches are applied during AFD config construction and + # worker startup, after vLLM-Ascend completes its platform initialization. from vllm.model_executor.models import ModelRegistry diff --git a/afd_plugin/compat/npu/__init__.py b/afd_plugin/compat/npu/__init__.py index 9e1a6c79..d5f5941b 100644 --- a/afd_plugin/compat/npu/__init__.py +++ b/afd_plugin/compat/npu/__init__.py @@ -19,6 +19,7 @@ has_afd_ascend_ops, ) from afd_plugin.compat.npu.runtime import ( + apply_afd_ascend_config_patch_if_needed, apply_afd_ascend_patches_if_needed, ascend_forward_context, fail_if_unsupported_npu_afd_features, @@ -27,6 +28,7 @@ ) __all__ = [ + "apply_afd_ascend_config_patch_if_needed", "apply_afd_ascend_patches_if_needed", "ascend_forward_context", "AFD_ASCEND_OPS_NAMESPACE", diff --git a/afd_plugin/compat/npu/runtime.py b/afd_plugin/compat/npu/runtime.py index 460dcf7e..68d46717 100644 --- a/afd_plugin/compat/npu/runtime.py +++ b/afd_plugin/compat/npu/runtime.py @@ -18,24 +18,31 @@ _PATCHES_APPLIED = False +def apply_afd_ascend_config_patch_if_needed() -> None: + """Apply patches required while vLLM builds an AFD NPU config.""" + + from afd_plugin.compat.patches.npu.ascend_platform import ( + apply_afd_ascend_dbo_config_patch, + ) + + if not apply_afd_ascend_dbo_config_patch(): + raise RuntimeError( + "AFD NPU DBO config patch requires vLLM-Ascend NPUPlatform", + ) + + def apply_afd_ascend_patches_if_needed() -> None: - """Apply plugin-owned, AFD-scoped Ascend patches.""" + """Apply plugin-owned runtime patches after Ascend initialization.""" global _PATCHES_APPLIED if _PATCHES_APPLIED: return - from afd_plugin.compat.patches.npu.ascend_platform import ( - apply_afd_ascend_dbo_config_patch, - ) from afd_plugin.compat.patches.npu.mla_graph import ( apply_afd_mla_graph_patch, ) - if not apply_afd_ascend_dbo_config_patch(): - raise RuntimeError( - "AFD NPU DBO config patch requires vLLM-Ascend NPUPlatform", - ) + apply_afd_ascend_config_patch_if_needed() if not apply_afd_mla_graph_patch(): raise RuntimeError( "AFD NPU MLA graph patch requires the vLLM-Ascend MLA resolver", @@ -44,6 +51,7 @@ def apply_afd_ascend_patches_if_needed() -> None: __all__ = [ + "apply_afd_ascend_config_patch_if_needed", "apply_afd_ascend_patches_if_needed", "ascend_forward_context", "fail_if_unsupported_npu_afd_features", diff --git a/afd_plugin/compat/patches/config_validation.py b/afd_plugin/compat/patches/config_validation.py index 0c4b9063..e99839a9 100644 --- a/afd_plugin/compat/patches/config_validation.py +++ b/afd_plugin/compat/patches/config_validation.py @@ -57,9 +57,11 @@ def create_engine_config( from vllm.platforms import current_platform if current_platform.device_type == "npu": - from afd_plugin.compat.npu import apply_afd_ascend_patches_if_needed + from afd_plugin.compat.npu import ( + apply_afd_ascend_config_patch_if_needed, + ) - apply_afd_ascend_patches_if_needed() + apply_afd_ascend_config_patch_if_needed() # ### PATCH END: AFD Ascend config patch ordering if not _should_relax_engine_args_backend(self): config = _original_create_engine_config( diff --git a/afd_plugin/v1/worker/npu/ffn_worker.py b/afd_plugin/v1/worker/npu/ffn_worker.py index e007be28..7f830faa 100644 --- a/afd_plugin/v1/worker/npu/ffn_worker.py +++ b/afd_plugin/v1/worker/npu/ffn_worker.py @@ -39,6 +39,11 @@ class AFDNPUFFNWorker(NPUWorker): afd_expected_role = "ffn" def __init__(self, *args: Any, **kwargs: Any) -> None: + # Import after vLLM-Ascend completes platform initialization. Importing + # its MoE modules from the general-plugin hook can race Ascend's own + # ops package initialization and leave DeviceOperator partially loaded. + import afd_plugin.compat.patches.npu.force_load_balance # noqa: F401 + apply_afd_ascend_patches_if_needed() super().__init__(*args, **kwargs) self._ffn_thread: threading.Thread | None = None diff --git a/tests/unit/compat/patches/test_config_validation.py b/tests/unit/compat/patches/test_config_validation.py index e1a7b240..cfead428 100644 --- a/tests/unit/compat/patches/test_config_validation.py +++ b/tests/unit/compat/patches/test_config_validation.py @@ -447,7 +447,7 @@ def test_config_validation_patch_auto_selects_afd_worker( arg_utils_module, config_module = _install_fake_vllm_config(monkeypatch) monkeypatch.setattr( npu_compat, - "apply_afd_ascend_patches_if_needed", + "apply_afd_ascend_config_patch_if_needed", lambda: None, ) config_module.VllmConfig.platform_worker_cls = platform_worker_cls @@ -493,7 +493,7 @@ def test_config_validation_installs_ascend_patch_only_on_npu(monkeypatch): calls = [] monkeypatch.setattr( npu_compat, - "apply_afd_ascend_patches_if_needed", + "apply_afd_ascend_config_patch_if_needed", lambda: calls.append("npu"), ) patch_module = _load_patch_module() @@ -540,7 +540,7 @@ def test_config_validation_patch_rejects_unsupported_auto_platform( arg_utils_module, config_module = _install_fake_vllm_config(monkeypatch) monkeypatch.setattr( npu_compat, - "apply_afd_ascend_patches_if_needed", + "apply_afd_ascend_config_patch_if_needed", lambda: None, ) config_module.VllmConfig.platform_worker_cls = platform_worker_cls From 1356b53cacf24f98a30e5fe8e3c9ff0a8988b9fa Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Wed, 5 Aug 2026 16:59:56 +0800 Subject: [PATCH 10/10] Fix NPU DBO stage metadata and review docs Signed-off-by: jiangkuaixue123 --- README.md | 8 +-- afd_plugin/connectors/README.md | 8 +-- afd_plugin/v1/worker/npu/ffn_model_runner.py | 53 ++++++++++--------- .../deepseek_v3_2/README.md | 44 ++++++--------- .../deepseek_v3_2/README.md | 16 +++--- tests/unit/v1/worker/test_npu_runtime.py | 50 +++++++++++++++++ 6 files changed, 111 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 9eaf4a63..fc76a8d3 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ See the [recipe index](recipe/README.md) for deployment and benchmark examples. | --- | --- | --- | --- | --- | --- | | `P2pNcclAFDConnector` | CUDA | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | FFN ranks are ordered before Attention ranks. `num_attention_ranks` must be greater than or equal to `num_ffn_ranks` and divisible by it. See the [DeepSeek V2 Lite recipe](recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md). | | `CAMP2pAFDConnector` | Ascend NPU | Decode | Sync | `FULL_DECODE_ONLY` ACL graph | Uses HCCL/CAMP2P custom ops. Ascend ops build by default on NPU platforms. See the [synchronous DeepSeek V3.2 recipe](recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md). | -| `CAMAsyncAFDConnector` | Ascend NPU | Prefill | Async | Not supported | Experimental. The v0.26 upgrade did not revalidate this path; the checked-in [PCP8 recipe](recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) records the earlier v0.19.1 experiment only. | +| `CAMAsyncAFDConnector` | Ascend NPU | Prefill | Async | Not supported | Validated on v0.26 without PCP. The checked-in [PCP8 recipe](recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) records the earlier v0.19.1rc1 experiment and must be used with the `release/v0.19.1rc1` branch. | Connector implementations are grouped by backend package: `afd_plugin.connectors.gpu` for GPU-only connectors, @@ -65,8 +65,8 @@ Known gaps: - GPU and NPU E2E tests are opt-in and require real hardware plus model weights. - GPU CUDA graph support is limited to `FULL_DECODE_ONLY`. - Native DBO is limited to exactly two ubatches. -- CAM async and PCP-based NPU model-runner-v1 deployments are not part of the - v0.26 validated runtime matrix. +- PCP-based NPU model-runner-v1 deployments from v0.19.1rc1 are not supported + on v0.26. ## Install @@ -94,7 +94,7 @@ The optional extra pins `vllm==0.26.0`. ### Ascend NPU installation -AFD's synchronous Ascend path is validated on openEuler 22.03 (aarch64) with +AFD's Ascend path is validated on openEuler 22.03 (aarch64) with Ascend 910C / Atlas A3. Install a compatible driver and firmware, and confirm the devices with `npu-smi info`. Use this source baseline: diff --git a/afd_plugin/connectors/README.md b/afd_plugin/connectors/README.md index dd040523..64bb0c27 100644 --- a/afd_plugin/connectors/README.md +++ b/afd_plugin/connectors/README.md @@ -8,10 +8,10 @@ AFD connector implementations are grouped by backend: by `afd_plugin.connectors.npu.camp2p`, and `CAMAsyncAFDConnector` is implemented by `afd_plugin.connectors.npu.async_cam`. -The vLLM 0.26 support matrix validates GPU `P2pNcclAFDConnector` and synchronous -NPU `CAMP2pAFDConnector`. `CAMAsyncAFDConnector` remains experimental and was -not revalidated during the v0.26 upgrade; its PCP8 recipe is a historical -v0.19.1 experiment. +The vLLM 0.26 support matrix validates GPU `P2pNcclAFDConnector` and NPU +`CAMP2pAFDConnector` and `CAMAsyncAFDConnector`. CAM async is validated without +PCP on v0.26; its PCP8 recipe is retained for v0.19.1rc1 and must be used with +the `release/v0.19.1rc1` branch. Shared connector contracts, metadata containers, factory registration, and backend-neutral helpers stay in `afd_plugin.connectors`. diff --git a/afd_plugin/v1/worker/npu/ffn_model_runner.py b/afd_plugin/v1/worker/npu/ffn_model_runner.py index 0444c653..83e81284 100644 --- a/afd_plugin/v1/worker/npu/ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/ffn_model_runner.py @@ -220,34 +220,35 @@ def _ffn_forward( num_stages=num_stages, ) stage_ids = sorted(int(stage_idx) for stage_idx in dp_metadata_list) or [0] - num_tokens_across_dp = _ffn_token_counts_across_ranks( - self.connector, - dp_metadata_list, - stage_ids[0], - fallback=self.max_num_tokens, - ) - num_tokens = _ffn_token_count_for_rank(self.connector, num_tokens_across_dp) rank_ffn_output = None - # Build DP-level token counts for vLLM's forward context. - # num_tokens_across_dp has ffn_size entries (AFD-level, one per - # role_rank = dp_rank * tp_size + tp_rank), but vLLM's DPMetadata - # expects dp_size entries where [dp_rank] equals batchsize. - dp_num_tokens_across_dp = _to_dp_level_token_counts( - num_tokens_across_dp, - dp_size=int(self.vllm_config.parallel_config.data_parallel_size), - ) - - with ascend_forward_context( - vllm_config=self.vllm_config, - afd_metadata=afd_metadata, - model_instance=self.model, - num_tokens=num_tokens, - num_tokens_across_dp=dp_num_tokens_across_dp, - aclgraph_runtime_mode=aclgraph_runtime_mode, - ) as forward_context: - for layer_idx in _ffn_layer_indices(self): - for stage_idx in stage_ids: + for layer_idx in _ffn_layer_indices(self): + for stage_idx in stage_ids: + num_tokens_across_dp = _ffn_token_counts_across_ranks( + self.connector, + dp_metadata_list, + stage_idx, + fallback=self.max_num_tokens, + ) + num_tokens = _ffn_token_count_for_rank( + self.connector, + num_tokens_across_dp, + ) + # DBO stages can have different token counts. Build a fresh + # Ascend context for each stage so its MC2 padding mask matches + # the hidden states received for that stage. + dp_num_tokens_across_dp = _to_dp_level_token_counts( + num_tokens_across_dp, + dp_size=int(self.vllm_config.parallel_config.data_parallel_size), + ) + with ascend_forward_context( + vllm_config=self.vllm_config, + afd_metadata=afd_metadata, + model_instance=self.model, + num_tokens=num_tokens, + num_tokens_across_dp=dp_num_tokens_across_dp, + aclgraph_runtime_mode=aclgraph_runtime_mode, + ) as forward_context: payload = self.connector.recv_attn_output( ubatch_idx=stage_idx, layer_idx=layer_idx, diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md index 475e6cca..5640ab59 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md @@ -1,13 +1,11 @@ # CAMAsyncAFDConnector For DeepSeek-V3.2 Recipe -> [!WARNING] -> This is a historical vLLM/vLLM-Ascend 0.19.1 PCP8 experiment. CAM async was -> not revalidated by the vLLM 0.26 upgrade, and vLLM-Ascend 0.26 removes PCP -> support from model runner v1. Keep these commands and measurements for -> provenance; do not treat them as a supported v0.26 deployment recipe. +> [!NOTE] +> This recipe targets vLLM/vLLM-Ascend `v0.19.1rc1`. Use the AFD Plugin +> branch `release/v0.19.1rc1` when running it. -This recipe records how DeepSeek-V3.2 was run with the AFD CAM async connector -on Ascend NPU in the legacy environment. +This recipe describes how to run DeepSeek-V3.2 with the AFD CAM async +connector on Ascend NPU. For the connector's complete configuration contract, rank derivation, data flow, native DBO distinction, and limitations, see the @@ -32,15 +30,11 @@ AFD provides the following backend-specific connectors: ## Image and Hardware Requirements - Hardware: Ascend 910C only. -- CANN: 9.0.1. -- Validated runtime image build: - `nightly-main-a3-openeuler-20260801230444_aarch64`. -- vLLM: v0.26.0 at commit `568afb3a1`. -- vLLM-Ascend: branch `releases/v0.26.0rc` at commit `80d8c194f`. +- Image: `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler`. -The nightly build name identifies the validation environment and is not -documented as a stable public image tag. Provision an equivalent runtime at -the commits above rather than using the former v0.19.1rc1 image command. +```bash +docker pull quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler +``` ## Installing Operator Packages @@ -69,12 +63,9 @@ for both sides. | `host` / `port` | Rendezvous address for the async CAM HCCL process group. Set `host` to the IP address of the node that owns attention rank 0; all attention and FFN workers must use the same `host` and `port`. | | `num_attention_ranks` | Total attention-side ranks in the AFD topology. In this recipe, `DP3PCP8` gives `3 * 8 = 24`. | | `num_ffn_ranks` | Total FFN-side ranks in the AFD topology. In this recipe, `EP8` gives `8`. | +| `afd_role_rank` | Role-local starting rank for the process. For attention workers, this is the data-parallel starting rank multiplied by `attn_ranks_per_dp`. | | `compute_gate_on_attention` | Runs MoE routing/gating on the attention side before dispatching activations to FFN ranks. | -AFD derives each process's role rank from its global DP rank and local PCP/TP -coordinates. Do not add a role-rank field to these configurations; -`data_parallel_start_rank` is already included in the global DP rank. - `connector_extra_config` carries CAM async-specific knobs: | Field | Meaning | @@ -271,9 +262,7 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -309,6 +298,7 @@ vllm serve /path/to/DeepSeek-V3.2 \ "port": 1239, "num_attention_ranks": 24, "num_ffn_ranks": 8, + "afd_role_rank": 0, "compute_gate_on_attention": true, "connector_extra_config": { "dynamicQuant": 1, @@ -337,9 +327,7 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -376,6 +364,7 @@ vllm serve /path/to/DeepSeek-V3.2 \ "port": 1239, "num_attention_ranks": 24, "num_ffn_ranks": 8, + "afd_role_rank": 16, "compute_gate_on_attention": true, "connector_extra_config": { "dynamicQuant": 1, @@ -404,9 +393,7 @@ export ASCEND_A3_ENABLE=1 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV -export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -432,6 +419,7 @@ vllm serve /path/to/DeepSeek-V3.2 \ "port": 1239, "num_attention_ranks": 24, "num_ffn_ranks": 8, + "afd_role_rank": 0, "compute_gate_on_attention": true, "connector_extra_config": { "dynamicQuant": 1, diff --git a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md index d9027bc6..f7c57522 100644 --- a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md @@ -1,5 +1,9 @@ # DeepSeek-V3.2 Synchronous Decode with CAMP2pAFDConnector on Ascend NPU +> [!NOTE] +> This recipe targets vLLM/vLLM-Ascend `v0.19.1rc1`. Use the AFD Plugin +> branch `release/v0.19.1rc1` when running it. + This recipe compares a conventional EP64 deployment with synchronous Attention-FFN Disaggregation (AFD) deployments for DeepSeek-V3.2 decode inference on Ascend NPUs. The AFD deployments use @@ -9,15 +13,15 @@ and FFN workers. ## Image and model requirements - Hardware: Ascend NPU, Atlas 900 A3 SuperPoD, 16 dies per node. -- Runtime: vLLM `0.26.0` with vLLM-Ascend source commit `80d8c194f`. +- Image: `quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler`. - Model: DeepSeek-V3.2 with W8A8 weights. - AFD Plugin: install this repository in the container. -Prepare the matching A3/openEuler environment using the -[vLLM-Ascend installation guide at `80d8c194f`](https://github.com/vllm-project/vllm-ascend/blob/80d8c194f7584b17fe08065ea99a130916f6b0e7/docs/source/installation.md), -then install this repository with `pip install -e . --no-build-isolation -v`. -The former `v0.19.1rc1-a3-openeuler` image is not a supported runtime for this -v0.26 recipe. +```bash +docker pull quay.io/ascend/vllm-ascend:v0.19.1rc1-a3-openeuler +cd /path/to/afd-plugin +pip install -e . --no-build-isolation -v +``` ## Topologies diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index 5015b1f5..b9d246d3 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -889,6 +889,56 @@ def test_npu_ffn_runner_executes_eager_ffn_step(monkeypatch): ] +def test_npu_ffn_runner_builds_forward_context_for_each_dbo_stage(monkeypatch): + _require_npu_runtime() + from afd_plugin.v1.worker.npu import ffn_model_runner + + context_calls = [] + + @contextmanager + def fake_ascend_forward_context(**kwargs): + context_calls.append(kwargs) + yield SimpleNamespace( + additional_kwargs={}, + dp_metadata=None, + all_moe_layers={}, + ) + + monkeypatch.setattr( + ffn_model_runner, + "ascend_forward_context", + fake_ascend_forward_context, + ) + runner = _new_ffn_runner() + runner.vllm_config = _vllm_config(role="ffn") + runner.connector = _FakeFFNConnector(attn_size=2, ffn_size=2) + runner.model = _FakeModel() + runner.num_layers = 1 + runner.max_num_tokens = 16 + runner.use_aclgraph = False + runner._acl_graphs = {} + for stage_idx, num_tokens in enumerate((6, 7)): + metadata = AFDTransferMetadata.create_attention_metadata( + layer_idx=0, + stage_idx=stage_idx, + seq_len=num_tokens, + ) + runner.connector.attn_outputs.append((f"hidden-{stage_idx}", metadata)) + + runner.execute_model( + dp_metadata_list={ + 0: _FakeDPMetadata([6]), + 1: _FakeDPMetadata([7]), + }, + ) + + assert [call["num_tokens"] for call in context_calls] == [6, 7] + assert [call["num_tokens_across_dp"].tolist() for call in context_calls] == [ + [6], + [7], + ] + + def test_npu_ffn_runner_dp_path_invokes_model_with_hidden_states_and_layer(monkeypatch): from afd_plugin.connectors.npu.async_cam import AFDAsyncTransferState