diff --git a/models/moonshotai/Kimi-K2.6.yaml b/models/moonshotai/Kimi-K2.6.yaml index 26baa61b..17dc5c49 100644 --- a/models/moonshotai/Kimi-K2.6.yaml +++ b/models/moonshotai/Kimi-K2.6.yaml @@ -3,7 +3,7 @@ meta: slug: "kimi-k2.6" provider: "Moonshot AI" description: "Open-source native multimodal agentic MoE model with vision-language understanding, tool calling, and thinking modes" - date_updated: 2026-05-14 + date_updated: 2026-07-11 difficulty: intermediate tasks: - multimodal @@ -12,6 +12,7 @@ meta: related_recipes: [] hardware: h200: verified + b300: verified gb200: verified mi300x: verified mi325x: verified @@ -19,15 +20,20 @@ meta: model: model_id: "moonshotai/Kimi-K2.6" - min_vllm_version: "0.19.1" + min_vllm_version: "0.25.0" docker_image: - nvidia: "vllm/vllm-openai:latest" + nvidia: "vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa" amd: "vllm/vllm-openai-rocm:nightly" + nightly_required: true + install: + docker: + note: "Recommended for the validated B300 NVFP4 path; the NVIDIA image is pinned to the tested nightly commit." + pip: + note: "The optimized B300 EAGLE3 and native CPU KV offload path requires post-v0.24.0 nightly support." architecture: moe parameter_count: "1T" active_parameters: "32B" context_length: 262144 - supports_dcp: true base_args: - "--trust-remote-code" @@ -48,6 +54,20 @@ features: args: - "--speculative-config" - '{"model":"lightseekorg/kimi-k2.6-eagle3-mla","method":"eagle3","num_speculative_tokens":3}' + hardware_overrides: + blackwell: + args: + - "--attention-backend" + - "TOKENSPEED_MLA" + - "--speculative-config" + - '{"method":"eagle3","model":"lightseekorg/kimi-k2.6-eagle3-mla","num_speculative_tokens":4,"rejection_sample_method":"synthetic","synthetic_acceptance_length":3.24}' + native_cpu_kv_offload: + description: "Offload prefix KV blocks to CPU DRAM with SimpleCPUOffloadConnector (8 GiB starter capacity; increase cpu_bytes_to_use for the host)" + args: + - "--disable-hybrid-kv-cache-manager" + - "--enable-prefix-caching" + - "--kv-transfer-config" + - '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":8589934592,"lazy_offload":false}}' text_only: description: "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel." args: @@ -60,6 +80,7 @@ features: opt_in_features: - text_only + - native_cpu_kv_offload # GB200's 4-GPU NVL4 trays keep encoder TP comm cheap — data-parallel encoder # isn't the default win it is on 8-GPU nodes. @@ -75,13 +96,27 @@ variants: nvfp4: model_id: "nvidia/Kimi-K2.6-NVFP4" precision: nvfp4 - vram_minimum_gb: 600 - description: "NVIDIA NVFP4 quantized weights for Blackwell GPUs (e.g. GB200)" + vram_minimum_gb: 715 + description: "NVIDIA ModelOpt NVFP4 checkpoint (~595 GB on disk); optimized for 4+ Blackwell GPUs" extra_args: - "--kv-cache-dtype" - "fp8" + - "--block-size" + - "64" + - "--gpu-memory-utilization" + - "0.90" + - "--compilation-config" + - '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + - "--max-cudagraph-capture-size" + - "2048" + - "--max-num-batched-tokens" + - "16384" + - "--stream-interval" + - "10" + - "--enable-prefix-caching" extra_env: VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" compatible_strategies: - single_node_tp @@ -96,7 +131,8 @@ compatible_strategies: hardware_overrides: blackwell: extra_args: - - "--attention-config.use_trtllm_ragged_deepseek_prefill=True" + - "--attention-config" + - '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' amd: # Verified on 8× MI300X / MI355X (MI325X listed as supported but not verified). extra_env: @@ -148,10 +184,92 @@ guide: | ## Prerequisites - - **vLLM version:** >= 0.19.1 + - **vLLM version:** >= 0.25.0 nightly for the optimized B300 EAGLE3 and native CPU + KV offload path documented below - **Hardware (INT4):** 8x H200 GPUs (verified), or equivalent aggregate VRAM (~640 GB) + - **Hardware (NVFP4):** 4x Blackwell GPUs; the optimized B300 path below was verified on + `vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa` - **AMD support:** 8x MI300X / MI325X / MI355X with ROCm 7.2.1 and Python 3.12 + ### NVIDIA B300: NVFP4 with Eagle3 + + The following text-only TP4 command mirrors the B300 configuration validated by + [InferenceX PR #2158](https://github.com/SemiAnalysisAI/InferenceX/pull/2158). It uses + the Kimi K2.6 Eagle3 MLA draft, TOKENSPEED_MLA attention, TRT-LLM ragged MLA prefill, + FP8 KV cache, and full-and-piecewise CUDA graphs. + + ```bash + export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + + vllm serve nvidia/Kimi-K2.6-NVFP4 \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --language-model-only \ + --kv-cache-dtype fp8 \ + --block-size 64 \ + --gpu-memory-utilization 0.90 \ + --attention-backend TOKENSPEED_MLA \ + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' \ + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ + --max-cudagraph-capture-size 2048 \ + --max-num-batched-tokens 16384 \ + --stream-interval 10 \ + --enable-prefix-caching \ + --speculative-config '{"method":"eagle3","model":"lightseekorg/kimi-k2.6-eagle3-mla","num_speculative_tokens":4,"rejection_sample_method":"synthetic","synthetic_acceptance_length":3.24}' + ``` + + ### Native CPU KV offload + + `SimpleCPUOffloadConnector` extends the prefix cache into host DRAM. The feature toggle + uses a conservative 8 GiB starter capacity. Size `cpu_bytes_to_use` for the host and divide + the aggregate budget across TP ranks. The verified B300 TP4 run used 1,199 GiB total + (299.75 GiB per rank): + + ```bash + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + CPU_OFFLOAD_BYTES=$((1199 * 1024 * 1024 * 1024)) + + vllm serve nvidia/Kimi-K2.6-NVFP4 \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --language-model-only \ + --kv-cache-dtype fp8 \ + --block-size 64 \ + --gpu-memory-utilization 0.90 \ + --attention-backend TOKENSPEED_MLA \ + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' \ + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ + --max-cudagraph-capture-size 2048 \ + --max-num-batched-tokens 16384 \ + --stream-interval 10 \ + --enable-prefix-caching \ + --speculative-config '{"method":"eagle3","model":"lightseekorg/kimi-k2.6-eagle3-mla","num_speculative_tokens":4,"rejection_sample_method":"synthetic","synthetic_acceptance_length":3.24}' \ + --disable-hybrid-kv-cache-manager \ + --kv-transfer-config "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":${CPU_OFFLOAD_BYTES},\"lazy_offload\":false}}" + ``` + + ### Decode context parallelism + + For higher concurrency, TP4/DCP4 was validated both with and without native CPU KV + offload. DCP is intentionally guide-only rather than exposed as a command-builder option. + Do not combine DCP with the Eagle3/TOKENSPEED_MLA flags above until + [vLLM PR #48180](https://github.com/vllm-project/vllm/pull/48180) lands. For the current + pinned image, remove `--attention-backend TOKENSPEED_MLA` and `--speculative-config`, then add: + + ```bash + --decode-context-parallel-size 4 + ``` + + The successful agentic sweep covered these B300 points: + + | Serving path | Parallelism | Native CPU KV offload | Tested concurrency | + |---|---:|:---:|---:| + | Eagle3 | TP8 | No | 1 | + | Eagle3 | TP4 | No | 2, 4, 8 | + | Eagle3 | TP4 | Yes | 8, 16, 32 | + | DCP | TP4/DCP4 | No | 32, 64, 128 | + | DCP | TP4/DCP4 | Yes | 64, 128, 256 | + ### AMD MI300X/MI325X On 8x MI300X or MI325X (`gfx942`), use the standard W4A16 MoE path with AITER @@ -252,11 +370,16 @@ guide: | for your specific hardware. - **Async scheduling:** Enabled by default for better throughput. Disable if you encounter issues, and file a bug report to vLLM. + - **Eagle3 with DCP:** The current pinned image does not support the combination. Disable + Eagle3/TOKENSPEED_MLA for DCP until vLLM PR #48180 is merged and available in the image. ## References - [Kimi-K2.6 on Hugging Face](https://huggingface.co/moonshotai/Kimi-K2.6) - [NVIDIA Kimi-K2.6-NVFP4 on Hugging Face](https://huggingface.co/nvidia/Kimi-K2.6-NVFP4) + - [InferenceX Kimi-K2.6 B300 agentic sweep](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/29158176591) + - [vLLM SimpleCPUOffloadConnector](https://github.com/vllm-project/vllm/blob/main/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py) + - [vLLM DCP + Eagle support PR](https://github.com/vllm-project/vllm/pull/48180) - [vLLM multimodal inputs guide](https://docs.vllm.ai/en/latest/features/multimodal_inputs.html) - [vLLM Expert Parallelism docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment.html) - [vLLM NixlConnector usage guide](https://docs.vllm.ai/en/latest/features/nixl_connector_usage.html)