Qualify and enable MTP 2 with paired workload evidence and fallback

This commit is contained in:
2026-09-18 22:37:54 +08:00
parent 06bcbcfd4e
commit 918b78a3b8
31 changed files with 3376 additions and 7 deletions
+3 -2
View File
@@ -30,10 +30,11 @@ services:
--host 0.0.0.0 --port 8000
--tensor-parallel-size 1 --dtype bfloat16
--engram-config '{"cpu_offload":true}'
--kv-cache-dtype fp8 --gpu-memory-utilization 0.96
--kv-cache-dtype fp8 --gpu-memory-utilization 0.985
--max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048
--enable-chunked-prefill --enable-prefix-caching
--compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1]}' --no-enable-flashinfer-autotune
--speculative-config '{"method":"mtp","num_speculative_tokens":2}'
--compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1,3]}' --no-enable-flashinfer-autotune
--load-format safetensors
--reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
--api-key "$$(cat /run/secrets/api-key)"