From 2c355817afe758b80a5f04c283c6f618d7f4eb85 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:57:36 -0700 Subject: [PATCH] Add NVIDIA Nemotron 3.5 Lightning recipe for H100, GB200 and DGX Spark Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- models/nvidia/nemotron-nano-3.5-ea2.yaml | 204 +++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 models/nvidia/nemotron-nano-3.5-ea2.yaml diff --git a/models/nvidia/nemotron-nano-3.5-ea2.yaml b/models/nvidia/nemotron-nano-3.5-ea2.yaml new file mode 100644 index 00000000..05204892 --- /dev/null +++ b/models/nvidia/nemotron-nano-3.5-ea2.yaml @@ -0,0 +1,204 @@ +meta: + title: "NVIDIA Nemotron 3.5 Lightning" + slug: "nemotron-3-5-lightning" + provider: "NVIDIA" + description: "NVIDIA Nemotron 3.5 Lightning hybrid Mamba-MoE model with NVFP4 (W4A4) and BF16 checkpoints, MTP and DFlash speculative decoding" + date_added: 2026-08-03 + date_updated: 2026-08-03 + difficulty: intermediate + tasks: + - text + related_recipes: + - "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + hardware: + h100: verified + gb200: verified + dgx_spark_gb10: verified + +model: + model_id: "nvidia/nemotron-nano-3.5-ea2" + min_vllm_version: "0.25.0" + nightly_required: true + architecture: moe + parameter_count: "30B" + active_parameters: "3B" + context_length: 262144 + base_args: + - "--trust-remote-code" + - "--mamba-backend" + - "flashinfer" + - "--mamba-cache-mode" + - "align" + base_env: + VLLM_USE_FASTOKENS: "1" + +features: + tool_calling: + description: "Qwen3 Coder tool-call parser with automatic tool choice" + args: + - "--enable-auto-tool-choice" + - "--tool-call-parser" + - "qwen3_coder" + mamba_fast_ssm_cache: + label: "Fast SSM Cache" + description: "FP16 Mamba SSM cache with stochastic rounding — faster decode than the FP32 default; the Philox rounds control the rounding stream" + args: + - "--mamba-ssm-cache-dtype" + - "float16" + - "--enable-mamba-cache-stochastic-rounding" + - "--mamba-cache-philox-rounds" + - "5" + spec_decoding: + description: "Speculative decoding — built-in MTP head or an externally trained DFlash draft model" + default_mode: mtp + modes: + mtp: + label: "MTP" + description: "Built-in Multi-Token Prediction head, 1 draft token, FlashInfer CUTLASS MoE kernels for the draft." + args: + - "--speculative-config" + - '{"method":"mtp","num_speculative_tokens":1,"moe_backend":"flashinfer_cutlass"}' + dflash: + label: "DFlash" + description: "DFlash drafting with an externally trained 3-token draft model; the served Nemotron checkpoint is unchanged." + variants: + - default + args: + - "--speculative-config" + - '{"method":"dflash","model":"nvidia/DFlash-Nemotron-3.5-Nano-MOPD-Intermediate","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + +opt_in_features: + - spec_decoding + +variants: + default: + model_id: "nvidia/nemotron-nano-3.5-ea2-W4A4-PTQ-20260723" + precision: nvfp4 + # NVFP4 expert weights are dequantized on the fly on Hopper, so this + # checkpoint is not Blackwell-only. + precision_hardware: + brand: NVIDIA + vram_minimum_gb: 18 + label: "NVFP4" + description: "ModelOpt PTQ NVFP4 (W4A4) checkpoint with FP8 KV cache — the default deployment target" + extra_args: + - "--kv-cache-dtype" + - "fp8" + hardware_overrides: + hopper: + # Hopper has no native NVFP4 path; the ModelOpt loader must be named explicitly. + extra_args: + - "--quantization" + - "modelopt_fp4" + bf16: + model_id: "nvidia/nemotron-nano-3.5-ea2" + precision: bf16 + vram_minimum_gb: 72 + description: "Full-precision BF16 reference checkpoint" + extra_args: + - "--moe-backend" + - "flashinfer_cutlass" + +compatible_strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + +hardware_overrides: {} + +strategy_overrides: {} + +guide: | + ## Overview + + Nemotron 3.5 Lightning is a hybrid Mamba-MoE model. It ships as two checkpoints: an + NVFP4 (W4A4) ModelOpt PTQ checkpoint — the default deployment target, which fits a + single DGX Spark (GB10) or a single H100 — and a BF16 reference checkpoint. + + Both are served with the FlashInfer Mamba backend and the `align` Mamba cache mode. + `VLLM_USE_FASTOKENS=1` is required. + + ## Prerequisites + + - Hardware: 1x DGX Spark (GB10), 1x H100, or 1x GB200 + - A vLLM nightly — the Mamba cache flags below are not in a stable release yet + + ```bash + docker pull vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de + ``` + + ## Launch commands + + NVFP4 on 1x H100: + + ```bash + export VLLM_USE_FASTOKENS=1 + + vllm serve nvidia/nemotron-nano-3.5-ea2-W4A4-PTQ-20260723 \ + --trust-remote-code \ + --quantization modelopt_fp4 \ + --kv-cache-dtype fp8 \ + --mamba-backend flashinfer \ + --mamba-cache-mode align \ + --mamba-ssm-cache-dtype float16 \ + --enable-mamba-cache-stochastic-rounding \ + --mamba-cache-philox-rounds 5 + ``` + + NVFP4 on 1x DGX Spark (GB10) with MTP: + + ```bash + export VLLM_USE_FASTOKENS=1 + + vllm serve nvidia/nemotron-nano-3.5-ea2-W4A4-PTQ-20260723 \ + --trust-remote-code \ + --kv-cache-dtype fp8 \ + --mamba-backend flashinfer \ + --mamba-cache-mode align \ + --mamba-ssm-cache-dtype float16 \ + --enable-mamba-cache-stochastic-rounding \ + --mamba-cache-philox-rounds 5 \ + --speculative-config '{"method":"mtp","num_speculative_tokens":1,"moe_backend":"flashinfer_cutlass"}' + ``` + + BF16 on 8x H100 with expert parallel and MTP: + + ```bash + export VLLM_USE_FASTOKENS=1 + + vllm serve nvidia/nemotron-nano-3.5-ea2 \ + --trust-remote-code \ + --tensor-parallel-size 8 \ + --enable-expert-parallel \ + --moe-backend flashinfer_cutlass \ + --mamba-backend flashinfer \ + --mamba-cache-mode align \ + --max-model-len 262144 \ + --speculative-config '{"method":"mtp","num_speculative_tokens":1,"moe_backend":"flashinfer_cutlass"}' + ``` + + Key flags: + - `--mamba-backend flashinfer` — required; the default backend has no kernel for this stack + - `--mamba-cache-mode align` — aligned Mamba state cache, needed for prefix caching + - `--mamba-ssm-cache-dtype float16` with stochastic rounding trades a little accuracy for decode speed; drop all three flags for the FP32 default + - `--quantization modelopt_fp4` — only needed on Hopper, where NVFP4 is not auto-detected + - DFlash drafting needs the separate `nvidia/DFlash-Nemotron-3.5-Nano-MOPD-Intermediate` draft checkpoint + + ## Known gaps + + - TP8 without expert parallel is not supported on the NVFP4 checkpoint. + + ## Benchmarking + + ```bash + vllm bench serve \ + --model nvidia/nemotron-nano-3.5-ea2-W4A4-PTQ-20260723 \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 32000 --random-output-len 2000 \ + --ignore-eos \ + --max-concurrency 256 \ + --num-prompts 1024 + ```