Enable validated prefix caching and record DGX Spark optimization benchmarks

2026-09-17 22:09:58 +08:00
parent 5f3030260e
commit 1e2f48d1a4
28 changed files with 707 additions and 8 deletions
@@ -0,0 +1,17 @@
+services:
+  vllm:
+    command:
+      - >-
+        exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47
+        --served-model-name qwen3.8-flash-next
+        --host 0.0.0.0 --port 8000
+        --tensor-parallel-size 1
+        --dtype bfloat16 --kv-cache-dtype auto
+        --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
+        --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
+        --enable-chunked-prefill --no-enable-prefix-caching
+        --speculative-config '{"method":"mtp","num_speculative_tokens":2}'
+        --enforce-eager --no-enable-flashinfer-autotune
+        --load-format safetensors
+        --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
+        --api-key "$$(cat /run/secrets/qwen_api_key)"