Set 128K eager baseline and organize deployment documentation

2026-09-18 01:33:57 +08:00
parent d7e1e745c3
commit e290793f2b
15 changed files with 98 additions and 11 deletions
@@ -8,7 +8,7 @@ services:
        --tensor-parallel-size 1
        --dtype bfloat16 --kv-cache-dtype auto
        --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
-        --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
+        --max-model-len 131072 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
        --enable-chunked-prefill --no-enable-prefix-caching
        --speculative-config '{"method":"mtp","num_speculative_tokens":2}'
        --enforce-eager --no-enable-flashinfer-autotune