Qualify and enable MTP 2 with paired workload evidence and fallback
This commit is contained in:
+3
-2
@@ -30,10 +30,11 @@ services:
|
||||
--host 0.0.0.0 --port 8000
|
||||
--tensor-parallel-size 1 --dtype bfloat16
|
||||
--engram-config '{"cpu_offload":true}'
|
||||
--kv-cache-dtype fp8 --gpu-memory-utilization 0.96
|
||||
--kv-cache-dtype fp8 --gpu-memory-utilization 0.985
|
||||
--max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048
|
||||
--enable-chunked-prefill --enable-prefix-caching
|
||||
--compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1]}' --no-enable-flashinfer-autotune
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":2}'
|
||||
--compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1,3]}' --no-enable-flashinfer-autotune
|
||||
--load-format safetensors
|
||||
--reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
|
||||
--api-key "$$(cat /run/secrets/api-key)"
|
||||
|
||||
Reference in New Issue
Block a user