Repository navigation
Expand file tree
/
Copy pathdocker-compose.prod.yml
More file actions
45 lines (44 loc) · 1.88 KB
/
Copy pathdocker-compose.prod.yml
File metadata and controls
45 lines (44 loc) · 1.88 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# Production-ish overlay for cave2 / chat.vr-bits.de integration.
# Base service comes from docker-compose.yml; CUDA settings from docker-compose.cuda.yml.
services:
granite-speech-api:
image: granite-speech-api:0.1.0-cave2
ports: !override
# Host port 8000 is already occupied on cave2.
# Bind to all interfaces so Open WebUI / HAProxy on the LAN can reach it.
- "${GRANITE_HOST_BIND:-0.0.0.0}:${GRANITE_API_PUBLIC_PORT:-8011}:8000"
environment:
GRANITE_DEVICE: cuda
GRANITE_DTYPE: ${GRANITE_DTYPE:-bfloat16}
GRANITE_IDLE_UNLOAD_SECONDS: ${GRANITE_IDLE_UNLOAD_SECONDS:-600}
GRANITE_CORS_ORIGINS: ${GRANITE_CORS_ORIGINS:-https://chat.vr-bits.de}
# Gated HF models (pyannote diarization, Cohere Transcribe). HF_TOKEN
# additionally lets huggingface_hub authenticate ALL downloads.
GRANITE_HF_TOKEN: ${GRANITE_HF_TOKEN:-}
HF_TOKEN: ${GRANITE_HF_TOKEN:-}
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-1}
NVIDIA_DRIVER_CAPABILITIES: compute,utility
# VRAM broker (see docs/network/docker.md): lease before a model is loaded, eviction via /v1/models/unload?if_idle=true
GRANITE_VRAM_BROKER_SOCK: /run/vram-broker/broker.sock
# Without this, unloading a model does not give the VRAM back (the allocator segment stays pinned by a few live
# tensors: 4.4 GB stayed after unload on 2026-10-12). Required for broker eviction to free anything.
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
volumes:
- /home/lefx/vram-broker/run:/run/vram-broker
deploy:
resources:
limits:
memory: 32G
reservations:
devices:
- driver: nvidia
device_ids: ["1"]
capabilities: [gpu]
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
networks:
default:
name: granite-speech-api