name: qwen38-flash-dgx-spark
services:
  vllm:
    image: local/qwen38-flash-spark:prefix-eager-0bfc7a15
    build:
      context: .
      args:
        BASE_IMAGE: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
    gpus: all
    ipc: host
    restart: unless-stopped
    mem_limit: 112g
    memswap_limit: 112g
    ports:
      - "${BIND_ADDRESS:-0.0.0.0}:${API_PORT:-8000}:8000"
    environment:
      HF_HUB_OFFLINE: "1"
      VLLM_PLE_CPU_OFFLOAD: "0"
      VLLM_PLE_MMAP_FAST_ROWS: "0"
      VLLM_PLE_MMAP_WORKERS: "32"
      VLLM_PLE_MMAP_PROMETHEUS: "0"
      VLLM_USE_V2_MODEL_RUNNER: "1"
      VLLM_WORKER_MULTIPROC_METHOD: spawn
      CUTE_DSL_ARCH: sm_121a
      TORCHINDUCTOR_COMPILE_THREADS: "2"
      HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
      HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
      NO_PROXY: localhost,127.0.0.1
    volumes:
      - ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface:ro
      - type: bind
        source: ${API_KEY_FILE:?Set API_KEY_FILE in .env}
        target: /run/secrets/qwen_api_key
        read_only: true
        bind:
          create_host_path: false
      - ./runtime-cache:/root/.cache/vllm
    entrypoint: ["/bin/bash", "-lc"]
    command:
      - >-
        exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47
        --served-model-name qwen3.8-flash-next
        --host 0.0.0.0 --port 8000
        --tensor-parallel-size 1
        --dtype bfloat16 --kv-cache-dtype auto
        --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80}
        --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048
        --enable-chunked-prefill --enable-prefix-caching
        --speculative-config '{"method":"mtp","num_speculative_tokens":2}'
        --enforce-eager --no-enable-flashinfer-autotune
        --load-format safetensors
        --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice
        --api-key "$$(cat /run/secrets/qwen_api_key)"
    healthcheck:
      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30m
  download:
    profiles: [tools]
    image: ${BASE_IMAGE:-vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28}
    environment:
      HTTP_PROXY: ${MODEL_HTTP_PROXY:-}
      HTTPS_PROXY: ${MODEL_HTTPS_PROXY:-}
      NO_PROXY: localhost,127.0.0.1
    volumes:
      - ${HF_CACHE_DIR:?Set HF_CACHE_DIR in .env}:/root/.cache/huggingface
      - ./scripts/download.py:/opt/download.py:ro
    entrypoint: [python3, -u, /opt/download.py]