Add reproducible DGX Spark deployment for Qwen3.8 Flash Next NVFP4

2026-09-17 16:45:50 +08:00
commit 5f3030260e
27 changed files with 1705 additions and 0 deletions
@@ -0,0 +1,70 @@
+name: qwen38-flash-dgx-spark
+services:
+  vllm:
+    image: local/qwen38-flash-spark:nightly-0bfc7a15
+    build:
+      context: .
+      args:
+        BASE_IMAGE: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
+    gpus: all
+    ipc: host
+    restart: unless-stopped
+    mem_limit: 112g
+    memswap_limit: 112g
+    ports:
+      - "${BIND_ADDRESS:-0.0.0.0}:${API_PORT:-8000}:8000"
+    environment:
+      HF_HUB_OFFLINE: "1"
+      VLLM_PLE_CPU_OFFLOAD: "0"
+      VLLM_PLE_MMAP_FAST_ROWS: "0"
+      VLLM_PLE_MMAP_WORKERS: "32"
+      VLLM_PLE_MMAP_PROMETHEUS: "0"
+      VLLM_USE_V2_MODEL_RUNNER: "1"
+      VLLM_WORKER_MULTIPROC_METHOD: spawn
+      CUTE_DSL_ARCH: sm_121a
+      TORCHINDUCTOR_COMPILE_THREADS: "2"
+      HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
+      HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
+      NO_PROXY: localhost,127.0.0.1
+    volumes:
+      - ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface:ro
+      - type: bind
+        source: ${API_KEY_FILE:?Set API_KEY_FILE in .env}
+        target: /run/secrets/qwen_api_key
+        read_only: true
+        bind:
+          create_host_path: false
+      - ./runtime-cache:/root/.cache/vllm
+    entrypoint: ["/bin/bash", "-lc"]
+    command:
+      - >-
+        exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47
+        --served-model-name qwen3.8-flash-next
+        --host 0.0.0.0 --port 8000
+        --tensor-parallel-size 1
+        --dtype bfloat16 --kv-cache-dtype auto
+        --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
+        --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
+        --enable-chunked-prefill --no-enable-prefix-caching
+        --speculative-config '{"method":"mtp","num_speculative_tokens":2}'
+        --enforce-eager --no-enable-flashinfer-autotune
+        --load-format safetensors
+        --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
+        --api-key "$$(cat /run/secrets/qwen_api_key)"
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30m
+  download:
+    profiles: [tools]
+    image: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
+    environment:
+      HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
+      HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
+      NO_PROXY: localhost,127.0.0.1
+    volumes:
+      - ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface
+      - ./scripts/download.py:/opt/download.py:ro
+    entrypoint: [python3, -u, /opt/download.py]