diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-maxseq2-mooncake-agentic.yaml new file mode 100644 index 000000000..195125d25 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-maxseq2-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# Measured p90-ITL Pareto family: OCI jobs 471335-471337 and 471341-471342, +# using one aggregate TP8 x DCP8 worker, DSpark K=4, and max-num-seqs 2. +name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-maxseq2-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Preserve the proven GB300 runner fabric rather than copying PR #211's + # single mlx5_0 rail, which is specific to the rail-isolated B300 fleet. + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + max-num-seqs: 2 + gpu-memory-utilization: 0.90 + max-num-batched-tokens: 8192 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=4 golden AL (3.36); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 1024 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml new file mode 100644 index 000000000..2b1c1d8cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml @@ -0,0 +1,135 @@ +# Measured p90-ITL Pareto family: OCI jobs 471604 (c32), 471605 (c48), and +# 471606 (c64), using one aggregate TP8 x DCP8 worker without a draft model. +name: "kimi-k3-vllm-agg-gb300-dcp8-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + gpu-memory-utilization: 0.90 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + max-cudagraph-capture-size: 1024 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml new file mode 100644 index 000000000..c79fbdd17 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -0,0 +1,204 @@ +# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. +# Measured Pareto family: OCI jobs 469790 (c32), 469796 (c48), and 469792 +# (c64), using DSpark K=4 with synthetic acceptance length 3.36. +name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + # Keep the colleague's measured 150 GB Mooncake segment setting. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + prefill_environment: &kimi_env + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # No-op in this recipe: shared-expert sequence-parallel sharding requires + # expert parallelism, which is disabled on both roles. + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. + # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no + # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On + # the NVSHMEM backend that reaches a process-global TeamManager singleton + # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a + # TP>1 engine whose workers span devices 0-3 dies with + # "Detected use of TeamManager on multiple devices." + # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job + # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem + # backend, which has no such singleton. + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads + # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the + # heap -- NixlConnector then dies with + # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND + # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the + # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls + # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, + # hpcx libs mapped = 0, 93-region registerMem OK on every rank. + NCCL_NET_PLUGIN: "none" + # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not + # allocate as host memory; REG_WHOLE=n keeps it from registering the whole + # enclosing region instead of the requested range. + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + # K3's linear-attention layers hold Mamba conv state. NixlConnector's + # 3-read conv transfer asserts the DS (dim, state_len) layout at worker + # init; without this both roles die with "requires DS conv state layout". + # Must match on prefill and decode, hence the shared anchor. + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + decode_environment: + <<: *kimi_env + + vllm_config: + prefill: + # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA + # KV cache across them instead of replicating it, giving the prefill role + # ~7.5x the KV pool it has under plain TP8. + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + max_num_batched_tokens: 16384 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see + # the header); DCP must likewise match at 8. + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + max-num-seqs: 256 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + # MODEL, topology, concurrency, duration, offload metadata, and result name + # are inherited from the generated InferenceMAX matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml new file mode 100644 index 000000000..6bd7a7fb6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml @@ -0,0 +1,204 @@ +# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. +# Measured Pareto family: OCI jobs 471040 (c1) and 471041 (c2), using DSpark +# K=7 with synthetic acceptance length 3.84. +name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + # Keep the colleague's measured 150 GB Mooncake segment setting. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + prefill_environment: &kimi_env + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # No-op in this recipe: shared-expert sequence-parallel sharding requires + # expert parallelism, which is disabled on both roles. + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. + # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no + # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On + # the NVSHMEM backend that reaches a process-global TeamManager singleton + # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a + # TP>1 engine whose workers span devices 0-3 dies with + # "Detected use of TeamManager on multiple devices." + # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job + # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem + # backend, which has no such singleton. + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads + # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the + # heap -- NixlConnector then dies with + # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND + # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the + # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls + # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, + # hpcx libs mapped = 0, 93-region registerMem OK on every rank. + NCCL_NET_PLUGIN: "none" + # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not + # allocate as host memory; REG_WHOLE=n keeps it from registering the whole + # enclosing region instead of the requested range. + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + # K3's linear-attention layers hold Mamba conv state. NixlConnector's + # 3-read conv transfer asserts the DS (dim, state_len) layout at worker + # init; without this both roles die with "requires DS conv state layout". + # Must match on prefill and decode, hence the shared anchor. + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + decode_environment: + <<: *kimi_env + + vllm_config: + prefill: + # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA + # KV cache across them instead of replicating it, giving the prefill role + # ~7.5x the KV pool it has under plain TP8. + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + max_num_batched_tokens: 16384 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see + # the header); DCP must likewise match at 8. + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + max-num-seqs: 256 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + # MODEL, topology, concurrency, duration, offload metadata, and result name + # are inherited from the generated InferenceMAX matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml new file mode 100644 index 000000000..c015f228d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -0,0 +1,204 @@ +# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. +# Measured p90-ITL Pareto point: OCI job 489701 (c48), using one DCP8 +# prefill worker, two DCP8 decode workers, DSpark K=4, and synthetic AL 3.36. +name: "kimi-k3-vllm-disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + # Keep the colleague's measured 150 GB Mooncake segment setting. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + prefill_environment: &kimi_env + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # No-op in this recipe: shared-expert sequence-parallel sharding requires + # expert parallelism, which is disabled on both roles. + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. + # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no + # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On + # the NVSHMEM backend that reaches a process-global TeamManager singleton + # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a + # TP>1 engine whose workers span devices 0-3 dies with + # "Detected use of TeamManager on multiple devices." + # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job + # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem + # backend, which has no such singleton. + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads + # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the + # heap -- NixlConnector then dies with + # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND + # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the + # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls + # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, + # hpcx libs mapped = 0, 93-region registerMem OK on every rank. + NCCL_NET_PLUGIN: "none" + # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not + # allocate as host memory; REG_WHOLE=n keeps it from registering the whole + # enclosing region instead of the requested range. + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + # K3's linear-attention layers hold Mamba conv state. NixlConnector's + # 3-read conv transfer asserts the DS (dim, state_len) layout at worker + # init; without this both roles die with "requires DS conv state layout". + # Must match on prefill and decode, hence the shared anchor. + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + decode_environment: + <<: *kimi_env + + vllm_config: + prefill: + # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA + # KV cache across them instead of replicating it, giving the prefill role + # ~7.5x the KV pool it has under plain TP8. + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + max_num_batched_tokens: 16384 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + enable-prefix-caching: true + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see + # the header); DCP must likewise match at 8. + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + enable-cumem-allocator: true + trust-remote-code: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + max-num-seqs: 256 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + # MODEL, topology, concurrency, duration, offload metadata, and result name + # are inherited from the generated InferenceMAX matrix. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8d55ad5d..77265b9cc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8575,6 +8575,116 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 16 dp-attn: true +# Kimi-K3 GB300 p90-ITL Pareto points. Each DCP8 worker spans two 4-GPU +# trays: 1P1D uses 16 GPUs and 1P2D uses 24 GPUs. +kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + # 0.1664 resolves to 150 GB of host DRAM per 4-GPU server node on + # cluster:gb300-nv, matching the recipe's Mooncake segment size. + - dram-utilization: 0.1664 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [64] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + +# Aggregate p90-ITL Pareto points. One TP8 x DCP8 worker spans two 4-GPU nodes +# and serves both phases; decode num-worker is zero so GPUs are counted once. +kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.1664 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-maxseq2-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + +# High-throughput aggregate p90-ITL Pareto points without speculative decoding +# or a max-num-seqs override. +kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.1664 + search-space: + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [48, 64, 70] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + + # Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 # GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, # ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73648d9b9..e834238a4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6066,6 +6066,16 @@ description: - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2619 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3 GB300 AgentX configurations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2637 - config-keys: - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8a..9ac6980d6 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -86,6 +86,9 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/Kimi-K2.5-NVFP4 export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH=/scratch/models/Kimi-K3 + export SRT_SLURM_MODEL_PREFIX="moonshotai/Kimi-K3" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our # Qwen3.5 sglang recipes (qwen3.5-fp4). @@ -97,7 +100,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, kimik3-fp4, qwen3.5-fp4, qwen3.5-fp8" exit 1 fi @@ -257,6 +260,9 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR"