diff --git a/models/MiniMaxAI/MiniMax-H3.yaml b/models/MiniMaxAI/MiniMax-H3.yaml index 51045b6a..01dc4307 100644 --- a/models/MiniMaxAI/MiniMax-H3.yaml +++ b/models/MiniMaxAI/MiniMax-H3.yaml @@ -4,7 +4,7 @@ meta: provider: "MiniMax" description: "Open-weight general-purpose multimodal generation model — jointly generates 24 FPS video with native stereo audio from text, image, video, and audio references, served via vLLM-Omni" date_added: 2026-08-02 - date_updated: 2026-08-06 + date_updated: 2026-08-08 difficulty: advanced tasks: - omni @@ -16,8 +16,9 @@ meta: gb200: verified b300: verified rtx_4090_2x: verified + rtx_pro_5000_4x: verified rtx_5090_2x: verified - rtx_pro_6000_2x: verified + rtx_pro_6000_2x: verified mi300x: verified mi325x: verified mi355x: verified @@ -28,15 +29,19 @@ model: docker_image: nvidia: "vllm/vllm-omni:minimax-h3" amd: "vllm/vllm-omni-rocm:minimax-h3" + # The published H3 images predate the modular pipeline merged in #5720. + # Docker binds current source read-only and prepends it to PYTHONPATH. + docker_source: + mount_path: "/path/to/vllm-omni" install: docker: - note: "Platform-specific H3 images bundle the model handlers and media dependencies. The refreshed official ROCm tag includes the Ref2VA fixes, TorchCodec, and FFmpeg." + note: "The H3 images bundle CUDA/ROCm and media dependencies, but predate the merged modular MiniMax-H3 pipeline. Clone current vLLM-Omni main; Docker mounts it read-only and prepends it to PYTHONPATH." pip: command: | uv venv source .venv/bin/activate uv pip install vllm==0.26.0 - note: "H3 also needs vLLM-Omni built from a checkout carrying the MiniMax H3 handlers — see the extra install block below." + note: "H3 also needs current vLLM-Omni main for the modular FL2VA/Ref2VA pipeline — see the extra install block below." architecture: dense parameter_count: "64B" active_parameters: "64B" @@ -72,24 +77,29 @@ model: omni: port: 8000 - # One server process loads exactly one checkpoint partition, so the served path - # swaps with the task: FL2VA serves t2va + fl2va, Ref2VA serves ref2va. H3 is - # served from the downloaded directory rather than the HF id — substitute your - # own --local-dir for /path/to/MiniMax-H3. + # The modular pipeline serves the checkpoint root and shares the encoder and + # VAEs across FL2VA/Ref2VA. Memory-constrained hardware overrides select one + # DiT with --task-type while retaining the same root model path. tasks: - id: t2v label: "T2VA · Text → Video+Audio" - model_id: "/path/to/MiniMax-H3/FL2VA" - vram_minimum_gb: 135 + model_id: "/path/to/MiniMax-H3" + vram_minimum_gb: 202 endpoint: "/v1/videos/sync" - description: "Text prompt → joint video + synchronized audio. Served by the FL2VA partition." + description: "Text prompt → joint video + synchronized audio through the shared modular service." hardware_overrides: + rtx_pro_5000_4x: + vram_minimum_gb: 288 + extra_args: ["--task-type", "fl2va"] rtx_pro_6000_2x: vram_minimum_gb: 192 + extra_args: ["--task-type", "fl2va"] rtx_5090_2x: vram_minimum_gb: 64 + extra_args: ["--task-type", "fl2va"] rtx_4090_2x: vram_minimum_gb: 48 + extra_args: ["--task-type", "fl2va"] curl: | curl -sS -X POST http://localhost:8000/v1/videos/sync \ -F 'prompt=In a snowy blue-purple forest, Ori carefully walks past a sleeping giant; footsteps crunch in the snow while the creature breathes and softly snorts.' \ @@ -114,17 +124,23 @@ omni: -o t2va.mp4 - id: i2v label: "FL2VA · First Frame → Video+Audio" - model_id: "/path/to/MiniMax-H3/FL2VA" - vram_minimum_gb: 135 + model_id: "/path/to/MiniMax-H3" + vram_minimum_gb: 202 endpoint: "/v1/videos/sync" - description: "First frame + prompt → video + audio, on the FL2VA partition. Omitting width/height keeps the first frame's aspect ratio at a 768 px short edge." + description: "First frame + prompt → video + audio through the shared modular service. Omitting width/height keeps the first frame aspect ratio at a 768 px short edge." hardware_overrides: + rtx_pro_5000_4x: + vram_minimum_gb: 288 + extra_args: ["--task-type", "fl2va"] rtx_pro_6000_2x: vram_minimum_gb: 192 + extra_args: ["--task-type", "fl2va"] rtx_5090_2x: vram_minimum_gb: 64 + extra_args: ["--task-type", "fl2va"] rtx_4090_2x: vram_minimum_gb: 48 + extra_args: ["--task-type", "fl2va"] curl: | export FIRST_FRAME=/path/to/fl2va_first_frame.png @@ -153,17 +169,23 @@ omni: -o fl2va.mp4 - id: ti2v label: "Ref2VA · Reference → Video+Audio" - model_id: "/path/to/MiniMax-H3/Ref2VA" - vram_minimum_gb: 168 + model_id: "/path/to/MiniMax-H3" + vram_minimum_gb: 202 endpoint: "/v1/videos/sync" - description: "Omni reference (image+audio, or one-or-more videos) → video + audio, on the Ref2VA partition. A full 15 s two-video request peaked at 166 GB on the main rank." + description: "Omni reference (image+audio, or one-or-more videos) → video + audio through the shared modular service. A full 15 s two-video request peaked at 166 GB on the main rank." hardware_overrides: + rtx_pro_5000_4x: + vram_minimum_gb: 288 + extra_args: ["--task-type", "ref2va"] rtx_pro_6000_2x: vram_minimum_gb: 192 + extra_args: ["--task-type", "ref2va"] rtx_5090_2x: vram_minimum_gb: 64 + extra_args: ["--task-type", "ref2va"] rtx_4090_2x: vram_minimum_gb: 48 + extra_args: ["--task-type", "ref2va"] curl: | # A) image + audio reference. Serve local assets first: # python -m http.server 8092 --bind 127.0.0.1 \ @@ -241,11 +263,20 @@ omni: dependencies: # Host-side prerequisite, not a pip install — the weights have to be on disk # before the container can bind-mount them, so this one shows under Docker too. - - note: "Gated checkpoint — request access on the model card, then download it locally. H3 is served from a local path, not the HF id: both partitions (FL2VA and Ref2VA) land as subdirectories and one server loads one of them" + - note: "Gated checkpoint — request access on the model card, then download it locally. Serve the root directory: both DiTs remain in subdirectories while the modular service shares the encoder and VAEs." command: "hf download MiniMaxAI/MiniMax-H3 --local-dir /path/to/MiniMax-H3" install_modes: [pip, docker] - - note: "pip path only (the Docker image already bundles it): MiniMax H3 support ships in vLLM-Omni, not the vllm wheel, so install it from a checkout. The [fa4] extra pulls the CuTe-DSL FlashAttention-4 kernels used on Blackwell" - command: "git clone https://github.com/vllm-project/vllm-omni.git && cd vllm-omni && uv pip install -e '.[fa4]'" + - note: "pip path only: install current vLLM-Omni main for the modular MiniMax-H3 pipeline." + command: "git clone --depth 1 https://github.com/vllm-project/vllm-omni.git /path/to/vllm-omni && cd /path/to/vllm-omni && uv pip install -e ." + install_modes: [pip] + - note: "Optional pip extra for the four-GPU B300/GB200 FLASH_ATTN profile. RTX PRO 5000 and the two-GPU RTX profiles use CUDNN_ATTN and do not need FA4." + command: "cd /path/to/vllm-omni && uv pip install -e '.[fa4]'" + install_modes: [pip] + brand: NVIDIA + optional: true + - note: "Docker path only: clone current vLLM-Omni main to the path mounted by the generated command so it overrides the older package in the image." + command: "git clone --depth 1 https://github.com/vllm-project/vllm-omni.git /path/to/vllm-omni" + install_modes: [docker] - note: "pip path only: ffmpeg and ffprobe must be on PATH for reference-video preparation and MP4 video+audio muxing. The official Docker images already bundle them." command: "sudo apt-get install -y ffmpeg" @@ -256,9 +287,25 @@ opt_in_features: [] variants: default: precision: bf16 - vram_minimum_gb: 135 - description: "BF16 — 66.3 GB DiT + 51.5 GB Qwen3-VL encoder + ~10.6 GB VAEs. 133 GB per-GPU engine peak measured on the validated 4×B300 no-offload profile; --text-encoder-tp-size 4 brings it to 103 GB" + vram_minimum_gb: 202 + description: "BF16 modular service — two 66.3 GB DiTs share the 51.5 GB Qwen3-VL encoder and ~10.6 GB VAEs. Use --task-type fl2va or ref2va on memory-constrained profiles to load one DiT." hardware_overrides: + rtx_pro_5000_4x: + extra_args: + - "--num-gpus" + - "4" + - "--tensor-parallel-size" + - "2" + - "--usp" + - "2" + - "--ring" + - "1" + - "--text-encoder-tp-size" + - "4" + - "--vae-patch-parallel-size" + - "4" + - "--diffusion-attention-backend" + - "CUDNN_ATTN" rtx_pro_6000_2x: extra_args: - "--num-gpus" @@ -361,18 +408,22 @@ guide: | - **Precise multimodal editing and control** — iterative refinement of characters, objects, scenes, sound, and rhythm with strong instruction following. - The checkpoint ships as **two independently served partitions**: + The checkpoint contains two DiT backbones that share the encoder and VAEs: - | Partition | Tasks | Conditioning | + | DiT | Tasks | Conditioning | |---|---|---| | `FL2VA` | `t2va`, `fl2va` | text only, or text + first/last frame | | `Ref2VA` | `ref2va` | text + omni references (images, video clips, audio clips) | - One server process loads one partition, so switching between T2VA/FL2VA and Ref2VA - means restarting the server against the other one. + Since vLLM-Omni [#5720](https://github.com/vllm-project/vllm-omni/pull/5720), serving + `/path/to/MiniMax-H3` initializes both DiTs by default, while the Qwen3-VL encoder and + video/audio VAEs stay shared. One server can therefore handle all three task types. + Add `--task-type fl2va` or `--task-type ref2va` only when the hardware profile needs to + load one DiT to save GPU or host memory. - Components (BF16): a 52-block joint video/audio DiT (66.3 GB), a Qwen3-VL layer-50 - text/vision encoder (51.5 GB), a video VAE (~10 GB) and an audio VAE (~0.6 GB). + Components (BF16): two 52-block joint video/audio DiTs (66.3 GB each), a shared + Qwen3-VL layer-50 text/vision encoder (51.5 GB), a video VAE (~10 GB), and an audio + VAE (~0.6 GB). ## Model specifications @@ -408,16 +459,17 @@ guide: | ## Prerequisites - H3 is served from a **local directory, not the HF id** — the checkpoint's two partitions - are subdirectories, and one server process loads one of them. Request access on the model - card (the repo is gated), then download it: + H3 is served from a **local directory, not the HF id**. The root contains both DiT + subdirectories plus the shared encoder and VAEs. Request access on the gated model card, + then download the complete repository: ```bash hf download MiniMaxAI/MiniMax-H3 --local-dir /path/to/MiniMax-H3 ``` - That yields `/path/to/MiniMax-H3/FL2VA` and `/path/to/MiniMax-H3/Ref2VA`. Substitute your - own `--local-dir` throughout the commands below. + That yields `/path/to/MiniMax-H3/FL2VA`, `/path/to/MiniMax-H3/Ref2VA`, and the + shared component directories. Serve `/path/to/MiniMax-H3` itself and substitute your own + `--local-dir` throughout the commands below. `ffmpeg` and `ffprobe` must be on `PATH` — they are used for reference-video preparation and MP4 muxing. @@ -426,36 +478,40 @@ guide: | ### Docker (recommended) - The image bundles the H3 handlers and the FlashAttention-4 kernels, so there is nothing - else to install: + The image bundles H3 handlers and media dependencies. The published tag predates the + modular pipeline merged in vLLM-Omni #5720, so clone current `main` before starting: ```bash docker pull vllm/vllm-omni:minimax-h3 + git clone --depth 1 https://github.com/vllm-project/vllm-omni.git /path/to/vllm-omni ``` - Because H3 is served from a local directory, the container needs that directory bound at - the same absolute path — the command builder's Docker mode emits the bind mount for you: + The command builder's Docker mode binds both the root checkpoint and source checkout, + then prepends the checkout to `PYTHONPATH`: ```bash - -v /path/to/MiniMax-H3/FL2VA:/path/to/MiniMax-H3/FL2VA:ro + -v /path/to/MiniMax-H3:/path/to/MiniMax-H3:ro + -v /path/to/vllm-omni:/path/to/vllm-omni:ro + -e PYTHONPATH=/path/to/vllm-omni ``` ### pip H3 support ships in vLLM-Omni rather than the `vllm` wheel, so the pip path needs a - source checkout: + current source checkout: ```bash uv venv source .venv/bin/activate uv pip install vllm==0.26.0 - git clone https://github.com/vllm-project/vllm-omni.git - cd vllm-omni - uv pip install -e '.[fa4]' + git clone --depth 1 https://github.com/vllm-project/vllm-omni.git /path/to/vllm-omni + cd /path/to/vllm-omni + uv pip install -e . ``` - The `[fa4]` extra installs the CuTe-DSL FlashAttention-4 kernels — CUDA-only, and used on - Blackwell. Drop the extra and the `FLASH_ATTN` backend falls back to FA3/FA2. + The RTX PRO 5000 and two-GPU RTX profiles use `CUDNN_ATTN` and need no + FlashAttention-4 install. For the B300/GB200 `FLASH_ATTN` profile, additionally run + `uv pip install -e '.[fa4]'`. ## Launch — two 24/32 GB GPUs with distributed layerwise offload @@ -480,14 +536,15 @@ guide: | # export DLO_RESIDENT_LAYERS=12 WIDTH=1024 HEIGHT=576 ``` - Start the FL2VA partition for T2VA and FL2VA: + Load only the FL2VA DiT on these memory-constrained profiles: ```bash CUDA_VISIBLE_DEVICES=0,1 \ VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \ - vllm serve /path/to/MiniMax-H3/FL2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni \ + --task-type fl2va \ --host 0.0.0.0 \ --port 8000 \ --trust-remote-code \ @@ -534,41 +591,28 @@ guide: | ffmpeg -v error -i minimax-h3-t2va.mp4 \ -map 0:v:0 -map 0:a:0 -f null - ``` - **Note:** If using 2× RTX 4090, the curl request must include -F 'aspect_ratio=16:9' to succeed. + **Note:** If using 2× RTX 4090, the curl request must include -F 'aspect_ratio=16:9' to succeed. A successful output has H.264 video at 24 FPS and stereo AAC audio at 32 kHz, and the - final `ffmpeg` command exits with status 0. One server loads one checkpoint partition; - stop it and restart the same command with `/path/to/MiniMax-H3/Ref2VA` before Ref2VA - requests. + final `ffmpeg` command exits with status 0. This capacity profile intentionally loads one + DiT. Restart the same root-model command with `--task-type ref2va` for Ref2VA; remove + `--task-type` only after confirming that the combined modular service fits the target. - The PR includes an all-task runner that starts both partitions in sequence, exercises - T2VA, FL2VA, image+audio Ref2VA, and two-video Ref2VA, validates every output stream, and - records live server and GPU-memory logs: - - ```bash - RUN_ROOT=/path/to/run-root \ - MODEL_ROOT=/path/to/MiniMax-H3 \ - GPU_IDS=0,1 \ - PROFILE=rtx5090 \ - bash recipes/MiniMaxAI/scripts/run_h3_2gpu_all_tasks.sh - ``` - - Use `PROFILE=rtx4090` for the 24 GB defaults. The script fails before inference when a - checkpoint partition, idle GPU, host-RAM budget, or shared FFmpeg/TorchCodec dependency - is missing. + The same root path is used for both task families; only `--task-type` changes on these + capacity profiles. ## Launch — four GPUs (validated best practice) - Measured on 4× NVIDIA B300: no CPU or layerwise offload, Ulysses SP degree 4, native - tiled VAE patch parallelism degree 4, regional `torch.compile` over the repeated DiT - blocks, FlashAttention, Ring and TP left at 1. + On B300/GB200, the no-offload root service uses Ulysses SP4, tiled VAE patch + parallelism 4, regional `torch.compile`, FlashAttention, and Ring/TP1. Both DiTs load + once while the encoder and VAEs remain shared. ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 \ FLASHINFER_DISABLE_VERSION_CHECK=1 \ VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \ - vllm serve /path/to/MiniMax-H3/FL2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni \ --host 0.0.0.0 \ --port 8000 \ @@ -582,8 +626,38 @@ guide: | --diffusion-attention-backend FLASH_ATTN ``` - Swap the served path to `/path/to/MiniMax-H3/Ref2VA` and restart to handle Ref2VA - requests instead. + No restart is required: `extra_params.task` routes T2VA/FL2VA requests to the FL2VA + DiT and Ref2VA requests to the Ref2VA DiT. + + ### RTX PRO 5000 Blackwell — four PCIe GPUs + + Select the `RTX PRO 5000 Blackwell 4x` hardware profile for the validated BF16, + no-offload topology: TP2 × Ulysses2, Ring1, text-encoder TP4, VAE patch parallel 4 + in tile mode, and `CUDNN_ATTN`. On the measured dual-socket host, + `CUDA_VISIBLE_DEVICES=0,2,1,3` kept the TP pairs on local PXB paths; check + `nvidia-smi topo -m` and adapt the order on other systems. + + This 72 GiB/GPU profile serves the root path but selects one DiT with `--task-type`. + The task picker adds `fl2va` for T2VA/FL2VA and `ref2va` for Ref2VA; loading both DiTs + together is not qualified on this memory-constrained profile. + + A five-step topology screen after warmup produced decodable H.264 video with stereo AAC: + + | Workload | Client E2E | Encode | Denoise | VAE decode | Peak/GPU | + |---|---:|---:|---:|---:|---:| + | T2VA, 124 frames, 1344×768 | 32.993 s | 0.034 s | 27.915 s | 4.307 s | 67.6 GiB | + | FL2VA first frame, same output shape | 30.130 s | 0.276 s | 24.396 s | 4.339 s | 67.6 GiB | + + These are topology-screening results, not 50-step throughput claims. Nsight Systems + aggregation for the same run was: + + | Workload | AllGather | Send/Recv | NCCL other | NCCL total | Dense FMHA | max-min/mean | + |---|---:|---:|---:|---:|---:|---:| + | T2VA | 0.93% | 8.53% | 14.63% | 24.09% | 31.15% | 1.11% | + | FL2VA | 0.89% | 7.26% | 15.76% | 23.91% | 32.53% | 0.13% | + + `NCCL other` covers NCCL kernels outside recognized AllGather and point-to-point + Send/Recv, including AllReduce, ReduceScatter, or Broadcast when present. ### AMD ROCm — four MI300X GPUs @@ -598,7 +672,7 @@ guide: | VLLM_ROCM_USE_AITER=1 \ VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \ - vllm serve /path/to/MiniMax-H3/FL2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni \ --host 0.0.0.0 \ --port 8000 \ @@ -633,7 +707,7 @@ guide: | VLLM_ROCM_USE_AITER=1 \ VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \ - vllm serve /path/to/MiniMax-H3/Ref2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni \ --host 0.0.0.0 \ --port 8000 \ @@ -648,7 +722,7 @@ guide: | --text-encoder-tp-size 4 ``` - Replace `Ref2VA` with `FL2VA` to serve text-to-video+audio or first-frame-to-video+audio. + This root-model command accepts T2VA, FL2VA, and Ref2VA requests without restarting. Three constraints on this configuration: @@ -673,8 +747,9 @@ guide: | CUDA_VISIBLE_DEVICES=0 \ VLLM_WORKER_MULTIPROC_METHOD=spawn \ VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \ - vllm serve /path/to/MiniMax-H3/FL2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni \ + --task-type fl2va \ --host 0.0.0.0 \ --port 8000 \ --trust-remote-code \ @@ -691,8 +766,9 @@ guide: | encoder process group: ```bash - vllm serve /path/to/MiniMax-H3/FL2VA \ + vllm serve /path/to/MiniMax-H3 \ --omni --trust-remote-code \ + --task-type fl2va \ --num-gpus 4 --usp 4 --ring 1 \ --text-encoder-tp-size 4 \ --vae-patch-parallel-size 4 --vae-parallel-mode tile --vae-use-tiling \ @@ -714,7 +790,7 @@ guide: | | Parameter | Recommended | Notes | |---|---|---| - | `task` | `t2va`, `fl2va`, `ref2va` | Passed inside `extra_params`; must match the served partition | + | `task` | `t2va`, `fl2va`, `ref2va` | Passed inside `extra_params`; must be compatible with `--task-type` when one is set | | `duration` | 4–15 s | Decimal seconds in `extra_params`; snapped to H3's legal `17n+5` frame count (15 s → 362 frames / 15.083 s) | | `fps` | `24` | Output FPS is fixed | | `num_inference_steps` | `50` | Matches the reference accuracy workloads | @@ -734,6 +810,9 @@ guide: | ## Validated performance (4×B300) + These measurements used task-specific single-DiT processes before the modular + combined-service merge; they remain the validated per-task operating points. + | Workload | Config | Result | |---|---|---| | FL2VA, 209 frames, 1248×768 (8.7 s) | no offload, U4, VPP4 tile, regional compile | **86.96 s** client E2E (±0.23 s) | @@ -788,7 +867,8 @@ guide: | ## Known limitations - - Each server process loads exactly **one** checkpoint partition. + - The modular root service loads both DiTs by default. Capacity profiles use + `--task-type fl2va` or `--task-type ref2va` and expose only that task family. - H3 executes **one generation request per diffusion batch** today. - The first request is a compile warmup only on the resident regional-compile profiles. The two-GPU DLO recipes use `--enforce-eager` and do not compile the DiT. @@ -815,6 +895,7 @@ guide: | ## References - [Model card](https://huggingface.co/MiniMaxAI/MiniMax-H3) + - [Modular FL2VA/Ref2VA pipeline (vLLM-Omni #5720)](https://github.com/vllm-project/vllm-omni/pull/5720) - [vLLM-Omni Video API](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/) - [vLLM-Omni diffusion parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/overview/) - [vLLM-Omni diffusion attention backends](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/attention_backends/) diff --git a/taxonomy.yaml b/taxonomy.yaml index df05800c..7fbb86a7 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -118,6 +118,19 @@ hardware_profiles: scalable: false restricted: true + # RTX PRO 5000 Blackwell — validated four-card PCIe profile. + # Restricted so it only appears for recipes with target-hardware evidence. + rtx_pro_5000_4x: + brand: NVIDIA + generation: blackwell + display_name: "RTX PRO 5000 Blackwell 4x" + description: "NVIDIA RTX PRO 5000 Blackwell · 4× 72 GB VRAM · PCIe, no NVLink" + gpu_count: 4 + vram_gb: 288 + multi_node: false + scalable: false + restricted: true + # RTX Pro 6000 8x — 8x RTX Pro 6000 GPUs system. # `restricted: true` keeps it out of the picker unless a recipe explicitly # lists it in `meta.hardware` (same pattern as the TPU profiles).