Document MTP 2 speed and memory tradeoff; retain stable default

2026-09-18 11:00:52 +08:00
parent 7e53821920
commit 80f70ec6f8
14 changed files with 758 additions and 0 deletions
@@ -0,0 +1,45 @@
+name: qwen38-flash-rtx6000d
+services:
+  vllm:
+    image: local/qwen38-flash-6000d:0bfc7a15
+    build: .
+    container_name: qwen38-flash-6000d
+    gpus: all
+    shm_size: 8g
+    mem_limit: 210g
+    memswap_limit: 210g
+    restart: unless-stopped
+    ports:
+      - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000"
+    environment:
+      HF_HUB_OFFLINE: "1"
+      VLLM_USE_V2_MODEL_RUNNER: "1"
+      VLLM_WORKER_MULTIPROC_METHOD: spawn
+      OMP_NUM_THREADS: "8"
+      TORCHINDUCTOR_COMPILE_THREADS: "2"
+      CUTE_DSL_ARCH: sm_120a
+    volumes:
+      - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro
+      - /data/flash-next/cache:/root/.cache
+      - ./secrets/api-key:/run/secrets/api-key:ro
+    entrypoint: ["/bin/bash", "-lc"]
+    command:
+      - >-
+        exec vllm serve /model
+        --served-model-name qwen3.8-flash-next
+        --host 0.0.0.0 --port 8000
+        --tensor-parallel-size 1 --dtype bfloat16
+        --engram-config '{"cpu_offload":true}'
+        --kv-cache-dtype fp8 --gpu-memory-utilization 0.96
+        --max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048
+        --enable-chunked-prefill --enable-prefix-caching
+        --compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1]}' --no-enable-flashinfer-autotune
+        --load-format safetensors
+        --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
+        --api-key "$$(cat /run/secrets/api-key)"
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30m