22 lines
1.7 KiB
YAML
22 lines
1.7 KiB
YAML
# Experimental: only promote after benchmark and correctness checks.
|
|
services:
|
|
vllm:
|
|
image: local/qwen38-flash-spark:graph2-0bfc7a15
|
|
environment:
|
|
VLLM_USE_BREAKABLE_CUDAGRAPH: "1"
|
|
command:
|
|
- >-
|
|
exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47
|
|
--served-model-name qwen3.8-flash-next
|
|
--host 0.0.0.0 --port 8000
|
|
--tensor-parallel-size 1
|
|
--dtype bfloat16 --kv-cache-dtype auto
|
|
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
|
|
--max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
|
|
--enable-chunked-prefill --no-enable-prefix-caching
|
|
--speculative-config '{"method":"mtp","num_speculative_tokens":2}'
|
|
--compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE","splitting_ops":["vllm::unified_attention_with_output","vllm::unified_mla_attention_with_output","vllm::mamba_mixer2","vllm::mamba_mixer","vllm::short_conv","vllm::qwen4_exp_ple_short_conv","vllm::qwen4_exp_qsa_with_output","vllm::linear_attention","vllm::qwen_gdn_attention_core","vllm::qwen_gdn_attention_core_fused_norm_packed","vllm::gdn_attention_core_xpu","vllm::olmo_hybrid_gdn_full_forward","vllm::sparse_attn_indexer","vllm::rocm_aiter_sparse_attn_indexer","vllm::deepseek_v4_attention","vllm::hpc_rope_norm_forward","vllm::unified_kv_cache_update","vllm::unified_mla_kv_cache_update","vllm::spark_ple_lookup"],"cudagraph_capture_sizes":[1,2,4,8,12]}' --no-enable-flashinfer-autotune
|
|
--load-format safetensors
|
|
--reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
|
|
--api-key "$$(cat /run/secrets/qwen_api_key)"
|