71 lines
2.7 KiB
YAML
71 lines
2.7 KiB
YAML
name: qwen38-flash-dgx-spark
|
|
services:
|
|
vllm:
|
|
image: local/qwen38-flash-spark:prefix-eager-0bfc7a15
|
|
build:
|
|
context: .
|
|
args:
|
|
BASE_IMAGE: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
|
|
gpus: all
|
|
ipc: host
|
|
restart: unless-stopped
|
|
mem_limit: 112g
|
|
memswap_limit: 112g
|
|
ports:
|
|
- "${BIND_ADDRESS:-0.0.0.0}:${API_PORT:-8000}:8000"
|
|
environment:
|
|
HF_HUB_OFFLINE: "1"
|
|
VLLM_PLE_CPU_OFFLOAD: "0"
|
|
VLLM_PLE_MMAP_FAST_ROWS: "0"
|
|
VLLM_PLE_MMAP_WORKERS: "32"
|
|
VLLM_PLE_MMAP_PROMETHEUS: "0"
|
|
VLLM_USE_V2_MODEL_RUNNER: "1"
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
CUTE_DSL_ARCH: sm_121a
|
|
TORCHINDUCTOR_COMPILE_THREADS: "2"
|
|
HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
|
|
HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
|
|
NO_PROXY: localhost,127.0.0.1
|
|
volumes:
|
|
- ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface:ro
|
|
- type: bind
|
|
source: ${API_KEY_FILE:?Set API_KEY_FILE in .env}
|
|
target: /run/secrets/qwen_api_key
|
|
read_only: true
|
|
bind:
|
|
create_host_path: false
|
|
- ./runtime-cache:/root/.cache/vllm
|
|
entrypoint: ["/bin/bash", "-lc"]
|
|
command:
|
|
- >-
|
|
exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47
|
|
--served-model-name qwen3.8-flash-next
|
|
--host 0.0.0.0 --port 8000
|
|
--tensor-parallel-size 1
|
|
--dtype bfloat16 --kv-cache-dtype auto
|
|
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
|
|
--max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
|
|
--enable-chunked-prefill --enable-prefix-caching
|
|
--speculative-config '{"method":"mtp","num_speculative_tokens":2}'
|
|
--enforce-eager --no-enable-flashinfer-autotune
|
|
--load-format safetensors
|
|
--reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
|
|
--api-key "$$(cat /run/secrets/qwen_api_key)"
|
|
healthcheck:
|
|
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 30m
|
|
download:
|
|
profiles: [tools]
|
|
image: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
|
|
environment:
|
|
HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
|
|
HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
|
|
NO_PROXY: localhost,127.0.0.1
|
|
volumes:
|
|
- ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface
|
|
- ./scripts/download.py:/opt/download.py:ro
|
|
entrypoint: [python3, -u, /opt/download.py]
|