nvidia-spark/Dockerfile.vllm.spark at main · Slach/nvidia-spark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# FROM vllm/vllm-openai:cu130-nightly-aarch64

# RUN pip install fastsafetensors

# paroquant + polarengine
# RUN apt-get update && apt-get install -y \
#    git \
#    ninja-build \
#    libcusparse-dev-13-0 \
#    libcublas-dev-13-0 \
#    libcurand-dev-13-0 \
#    libcusolver-dev-13-0 \
#    && rm -rf /var/lib/apt/lists/*

# RUN pip install paroquant[vllm] \
#    && pip install --upgrade transformers \
#    && pip install git+https://github.com/caiovicentino/polarengine-vllm.git

# COPY docker/paroquant/patch.py /tmp/patch.py
# RUN python3 /tmp/patch.py && rm /tmp/patch.py

# RUN python3 -c "import paroquant.kernels.cuda"

ARG UBUNTU_VERSION=24.04
ARG CUDA_VERSION=13.3.1
ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
ARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}

FROM ${BASE_CUDA_DEV_CONTAINER} AS build

RUN apt-get update && apt-get install -y \
    python3-pip python3.12-dev python3.12-venv pipx \
    build-essential ninja-build cmake git curl bash ccache \
    && rm -rf /var/lib/apt/lists/*

SHELL ["/bin/bash", "-c"]
ENV PATH="/root/.local/bin:${PATH}"
RUN pipx install uv

# only requirements first: deps + cubins + AOT layers stay cached across vllm source edits
COPY requirements/ /vllm/requirements/
WORKDIR /vllm/

ENV CUDA_HOME=/usr/local/cuda-13.3 \
    TORCH_CUDA_ARCH_LIST="12.1a" \
    FLASHINFER_CUDA_ARCH_LIST="12.1a" \
    FLASHINFER_CUBIN_DOWNLOAD_THREADS=10 \
    FLASHINFER_LOGGING_LEVEL=WARNING \
    VLLM_TARGET_DEVICE=cuda \
    VLLM_CUDA_VERSION="13.3" \
    UV_CONCURRENT_DOWNLOADS=3 \
    UV_HTTP_RETRIES=10

RUN --mount=type=cache,target=/root/.cache/ \
    uv venv --python python3.12 && \
    source ./.venv/bin/activate && \
    uv pip install -r requirements/build/cuda.txt -r requirements/cuda.txt --torch-backend cu130 && \
    uv pip install fastsafetensors runai-model-streamer

RUN --mount=type=cache,target=/root/.cache/ \
    source ./.venv/bin/activate && \
    export MAX_JOBS=$(nproc) && \
    /vllm/.venv/bin/python3.12 -m flashinfer.aot && \
    HTTP_PROXY=http://192.168.3.3:9998 /vllm/.venv/bin/python3.12 -m flashinfer --download-cubin || true

# source edits invalidate only from here down; ccache (cache mount) makes recompile fast
COPY ./ /vllm/

RUN --mount=type=cache,target=/root/.cache/ \
    source ./.venv/bin/activate && \
    export MAX_JOBS=$(nproc) CCACHE_DIR=/root/.cache/ccache CCACHE_COMPILERCHECK=content && \
    export CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache" && \
    uv pip install . --no-build-isolation -v

FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime

RUN apt update && \
    apt install -y python3.12 python3.12-venv python3.12-dev gcc g++ ninja-build curl && \
    apt install -y cuda-nvcc-13-3 cuda-libraries-dev-13-3 && \
    rm -rf /var/lib/apt/lists/*

COPY --from=build /vllm/.venv /vllm/.venv
ENV PATH="/vllm/.venv/bin:${PATH}"
ENV VLLM_USAGE_SOURCE=docker
# for properly fastsafetensors initialization
# RUN ln -s /usr/local/cuda/lib64/libcudart.so.13 /usr/local/cuda/lib64/libcudart.so

### Flint: Recursive Flint support (vllm PR #62938)
# flint-arm64-cu130 was built from PR commit db02bc9f, before 826fd3f added gguf>=0.19.0
# to the image - Flint NVFP4 GGUF loading dies on "requires the gguf package" (#63020).
# One fast layer on top of the pulled image; drop this target when the tag is rebuilt
# after the PR merges. Build: build-vllm-cuda13-docker.sh (vllm:flint tag)
FROM vllm/vllm-openai:flint-arm64-cu130 AS flint
RUN pip install --no-cache-dir "gguf>=0.19.0"

### Chisel: chisel-nvfp4-dflash2-replay recipe (yuanlin2004/chisel-spark-recipes)
# 65.1 t/s decode Qwen3.8-27B recipe: the pinned spark-arena vLLM nightly
# (0.31.1rc1.dev380) + the chisel-passes plugin (GDN state replay + reduced draft
# vocabulary, lossless spec-decode passes). Pinned to the recipe repo commit 84feafd.
# Same pip install as recipes/qwen3.8-27b/mods/chisel-passes/run.sh does at start.
# Build: build-vllm-cuda13-docker.sh (vllm:chisel tag)
FROM ghcr.io/spark-arena/dgx-vllm-eugr-nightly@sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0 AS chisel
RUN curl -fsSL https://github.com/yuanlin2004/chisel-spark-recipes/archive/84feafddff1e834bd16b849c827eed11676e5934.tar.gz | tar xz -C /tmp \
    && pip install --no-cache-dir --no-deps --no-build-isolation /tmp/chisel-spark-recipes-84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/mods/chisel-passes \
    && rm -rf /tmp/chisel-spark-recipes-*