name: qwen38-flash-rtx6000d
services:
  vllm:
    image: local/qwen38-flash-6000d:0bfc7a15
    build: .
    container_name: qwen38-flash-6000d
    gpus: all
    shm_size: 8g
    mem_limit: 210g
    memswap_limit: 210g
    restart: "no"
    ports:
      - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000"
    environment:
      HF_HUB_OFFLINE: "1"
      VLLM_USE_V2_MODEL_RUNNER: "1"
      VLLM_WORKER_MULTIPROC_METHOD: spawn
      OMP_NUM_THREADS: "8"
      TORCHINDUCTOR_COMPILE_THREADS: "2"
      CUTE_DSL_ARCH: sm_120a
    volumes:
      - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro
      - /data/flash-next/cache:/root/.cache
      - ./secrets/api-key:/run/secrets/api-key:ro
    entrypoint: ["/bin/bash", "-lc"]
    command:
      - >-
        exec vllm serve /model
        --served-model-name qwen3.8-flash-next
        --host 0.0.0.0 --port 8000
        --tensor-parallel-size 1 --dtype bfloat16
        --engram-config '{"cpu_offload":true}'
        --kv-cache-dtype fp8 --gpu-memory-utilization 0.96
        --max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048
        --enable-chunked-prefill --enable-prefix-caching
        --enforce-eager --no-enable-flashinfer-autotune
        --load-format safetensors
        --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
        --api-key "$$(cat /run/secrets/api-key)"
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30m