47 lines
1.7 KiB
YAML
47 lines
1.7 KiB
YAML
name: qwen38-flash-rtx6000d
|
|
services:
|
|
vllm:
|
|
image: local/qwen38-flash-6000d:0bfc7a15
|
|
build: .
|
|
container_name: qwen38-flash-6000d
|
|
gpus: all
|
|
shm_size: 8g
|
|
mem_limit: 210g
|
|
memswap_limit: 210g
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000"
|
|
environment:
|
|
HF_HUB_OFFLINE: "1"
|
|
VLLM_USE_V2_MODEL_RUNNER: "1"
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
OMP_NUM_THREADS: "8"
|
|
TORCHINDUCTOR_COMPILE_THREADS: "2"
|
|
CUTE_DSL_ARCH: sm_120a
|
|
volumes:
|
|
- /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro
|
|
- /data/flash-next/cache:/root/.cache
|
|
- ./secrets/api-key:/run/secrets/api-key:ro
|
|
entrypoint: ["/bin/bash", "-lc"]
|
|
command:
|
|
- >-
|
|
exec vllm serve /model
|
|
--served-model-name qwen3.8-flash-next
|
|
--host 0.0.0.0 --port 8000
|
|
--tensor-parallel-size 1 --dtype bfloat16
|
|
--engram-config '{"cpu_offload":true}'
|
|
--kv-cache-dtype fp8 --gpu-memory-utilization 0.985
|
|
--max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048
|
|
--enable-chunked-prefill --enable-prefix-caching
|
|
--speculative-config '{"method":"mtp","num_speculative_tokens":2}'
|
|
--compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1,3]}' --no-enable-flashinfer-autotune
|
|
--load-format safetensors
|
|
--reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
|
|
--api-key "$$(cat /run/secrets/api-key)"
|
|
healthcheck:
|
|
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 30m
|