From 7e53821920c6229c9ff7d8b620126e6812138df0 Mon Sep 17 00:00:00 2001 From: ruihan Date: Fri, 18 Sep 2026 10:47:26 +0800 Subject: [PATCH] Deploy audited RTX 6000D Flash-Next 128K multimodal baseline --- .dockerignore | 4 + .env.example | 3 + .gitignore | 14 + Dockerfile | 4 + README.md | 75 ++ audit/expected-source-hashes.json | 7 + audit/results.md | 41 + audit/runtime/acceptance-128k-fp8-eager.jsonl | 4 + audit/runtime/acceptance-128k-graph.jsonl | 4 + audit/runtime/acceptance-32k.jsonl | 4 + audit/runtime/acceptance-final-ready.jsonl | 4 + audit/runtime/base-image.txt | 1 + audit/runtime/benchmark-eager.jsonl | 4 + audit/runtime/benchmark-graph.jsonl | 4 + audit/runtime/benchmark-summary.json | 12 + audit/runtime/component-status | 1 + audit/runtime/compose-128k-fp8-eager.yaml | 45 + audit/runtime/compose-128k-graph.yaml | 45 + audit/runtime/compose-32k-bf16.yaml | 45 + audit/runtime/context-128k-fp8-eager.jsonl | 2 + audit/runtime/context-128k-graph.jsonl | 2 + audit/runtime/context-32k.jsonl | 2 + audit/runtime/expected-weights.json | 1 + audit/runtime/final-status.json | 20 + audit/runtime/h2d.json | 1 + audit/runtime/multimodal-128k-fp8-eager.jsonl | 2 + audit/runtime/multimodal-128k-graph.jsonl | 2 + audit/runtime/old-containers.txt | 3 + audit/runtime/ple-tests.log | 12 + audit/runtime/qsa-tests.log | 12 + audit/runtime/resources.jsonl | 788 ++++++++++++++++++ audit/runtime/source-hashes.json | 1 + audit/runtime/weights-verification.jsonl | 11 + audit/runtime/weights-verified | 1 + audit/source-review.md | 32 + compose.yaml | 45 + docs/lessons.md | 38 + patches/Apache-2.0.txt | 201 +++++ patches/fix_prefix_alignment.py | 26 + patches/prefix-edits.json | 14 + scripts/acceptance.py | 65 ++ scripts/benchmark.py | 33 + scripts/long_context.py | 57 ++ scripts/monitor.py | 22 + scripts/multimodal.py | 31 + 45 files changed, 1745 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 audit/expected-source-hashes.json create mode 100644 audit/results.md create mode 100644 audit/runtime/acceptance-128k-fp8-eager.jsonl create mode 100644 audit/runtime/acceptance-128k-graph.jsonl create mode 100644 audit/runtime/acceptance-32k.jsonl create mode 100644 audit/runtime/acceptance-final-ready.jsonl create mode 100644 audit/runtime/base-image.txt create mode 100644 audit/runtime/benchmark-eager.jsonl create mode 100644 audit/runtime/benchmark-graph.jsonl create mode 100644 audit/runtime/benchmark-summary.json create mode 100644 audit/runtime/component-status create mode 100644 audit/runtime/compose-128k-fp8-eager.yaml create mode 100644 audit/runtime/compose-128k-graph.yaml create mode 100644 audit/runtime/compose-32k-bf16.yaml create mode 100644 audit/runtime/context-128k-fp8-eager.jsonl create mode 100644 audit/runtime/context-128k-graph.jsonl create mode 100644 audit/runtime/context-32k.jsonl create mode 100644 audit/runtime/expected-weights.json create mode 100644 audit/runtime/final-status.json create mode 100644 audit/runtime/h2d.json create mode 100644 audit/runtime/multimodal-128k-fp8-eager.jsonl create mode 100644 audit/runtime/multimodal-128k-graph.jsonl create mode 100644 audit/runtime/old-containers.txt create mode 100644 audit/runtime/ple-tests.log create mode 100644 audit/runtime/qsa-tests.log create mode 100644 audit/runtime/resources.jsonl create mode 100644 audit/runtime/source-hashes.json create mode 100644 audit/runtime/weights-verification.jsonl create mode 100644 audit/runtime/weights-verified create mode 100644 audit/source-review.md create mode 100644 compose.yaml create mode 100644 docs/lessons.md create mode 100644 patches/Apache-2.0.txt create mode 100644 patches/fix_prefix_alignment.py create mode 100644 patches/prefix-edits.json create mode 100644 scripts/acceptance.py create mode 100644 scripts/benchmark.py create mode 100644 scripts/long_context.py create mode 100644 scripts/monitor.py create mode 100644 scripts/multimodal.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d92dcc9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +* +!Dockerfile +!patches/ +!patches/** diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5a100fb --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +# LAN service; use 127.0.0.1 for host-only access +BIND_ADDRESS=0.0.0.0 +API_PORT=8000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96f6302 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.env +secrets/ +runtime-cache/ +__pycache__/ +*.pyc +# Host-only data and credentials +models/ +cache/ +*.safetensors +*.gguf +*.pem +*.key +.env.* +!.env.example diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b778106 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28 +COPY patches/ /opt/6000d-patches/ +RUN python3 /opt/6000d-patches/fix_prefix_alignment.py +LABEL org.opencontainers.image.description="Audited NVIDIA Flash-Next baseline for RTX 6000D" diff --git a/README.md b/README.md new file mode 100644 index 0000000..03e3de4 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# Qwen3.8 Flash-Next on RTX 6000D + +2026-09-18 已完成部署和验收。保留文字、图片、视频能力；采用 NVIDIA NVFP4 权重、原生 FP8 PLE 主机内存卸载、FP8 KV、128K 上下文及 FULL CUDA Graph。单个活动请求，MTP 暂不启用。 + +社区代码经过审计后仅作为参考，没有执行社区启动脚本或使用社区镜像。审计范围和取舍见 [源码审计](audit/source-review.md)，实测数据见 [验收结果](audit/results.md)。 + +## 仓库导航 + +- `compose.yaml`、`Dockerfile`：已验收部署配置与固定镜像。 +- `patches/`：最小补丁、精确源码保护及许可证。 +- [部署心得](docs/lessons.md)：取舍、踩坑、优化方向及升级方法。 +- `scripts/`：功能、上下文、速度和资源验证工具。 +- `audit/runtime/`：2026-09-18 历史原始回执，不代表实时状态。 + +本仓库不保存 SSH 密码、API key、私有代理配置、模型权重或运行缓存。 + +## 调用 + +- OpenAI 兼容 Base URL：`http://172.16.104.92:8000/v1` +- 模型名：`qwen3.8-flash-next` +- API key：服务器 `/data/flash-next/project/secrets/api-key`，不纳入项目。 +- 健康接口：`/health`；业务接口需要 Bearer API key。 +- 131072 token 是输入、思考和最终输出的总预算。图片和视频也消耗上下文和计算资源。 + +在服务器执行以下示例，不会将密钥写进脚本： + +```bash +cd /data/flash-next/project +python3 - <<'PY' +import json, urllib.request +from pathlib import Path +body = {'model': 'qwen3.8-flash-next', 'messages': [{'role': 'user', 'content': '你好，请介绍一下自己。'}], 'max_tokens': 1024, 'reasoning_effort': 'low'} +req = urllib.request.Request('http://127.0.0.1:8000/v1/chat/completions', data=json.dumps(body).encode(), headers={'Content-Type': 'application/json', 'Authorization': 'Bearer '+Path('secrets/api-key').read_text().strip()}) +print(json.load(urllib.request.urlopen(req, timeout=180))) +PY +``` + +图片和视频请求示例见 `scripts/multimodal.py`，使用 `image_url` 和 `video_url` 内容块。 + +## 固定版本 + +- 官方 vLLM commit：`0bfc7a15d095fe83ecc82b50561a93c177fece2d`，镜像 digest 写在 Dockerfile。 +- NVIDIA 模型 revision：`fc694b54fb0174e0913e6adf86691ef85a4ead47`。 +- 11 个 safetensors 文件全部通过 SHA256 校验。 +- PLE 保留原生 FP8 数据及缩放，常驻锁页 CPU 内存，通过 UVA 查表；本配置不使用磁盘 mmap PLE。 +- 额外补丁仅两处 Mamba prefix block alignment 修复；修改前核对精确源码哈希，保留出处和许可证。 + +不要换成浮动 nightly 后继续沿用本项目的验收结论。 + +## 运维 + +服务器目录 `/data/flash-next/` 下，`project/` 是部署项目，`models/` 为权重，`cache/` 为缓存，`audit/` 和 `logs/` 保存验证证据。 + +```bash +cd /data/flash-next/project +docker compose build +docker compose up -d +docker compose logs --tail 100 -f +# 主动停止；unless-stopped 会尊重手动停止 +# docker compose stop +``` + +Compose 默认绑定回环地址；服务器 `.env` 设置 `BIND_ADDRESS=0.0.0.0` 提供内网访问。API key 文件必须存在。新环境可参考 `.env.example`，在 `secrets/` 生成随机密钥，目录权限 0700、文件权限 0600。镜像和模型下载代理属于机器私有配置，不写入项目。 + +已移除本机旧 MinerU、旧 27B 容器、相关旧镜像和模型目录。保留操作系统、SSH、网络和已修复的 NVIDIA 驱动。DGX Spark 服务配置未改动。 + +## 复测 + +```bash +python3 scripts/acceptance.py --output /data/flash-next/audit/acceptance-new.jsonl +python3 scripts/long_context.py --tokens 128000 --output /data/flash-next/audit/context-new.jsonl +python3 scripts/benchmark.py --output /data/flash-next/audit/benchmark-new.jsonl +``` + +脚本拒绝覆盖同名结果。长上下文验证为固定位置检索及前缀复用；多模态验证为合成图片和短视频；不代表综合质量评测或长期压力测试。当前基线优先单请求和多模态能力，额外并发、MTP、更长视频应另行测试。 diff --git a/audit/expected-source-hashes.json b/audit/expected-source-hashes.json new file mode 100644 index 0000000..1cc2801 --- /dev/null +++ b/audit/expected-source-hashes.json @@ -0,0 +1,7 @@ +{ + "models/qwen4_exp/nvidia/ngram_embedding.py": "01c2cb7c691c9a8a448e57619fdb97bfe1c3230e7be3c92dd4642d3f7173a499", + "models/qwen4_exp/nvidia/qsa.py": "f7307663d341e3fdf5ee2ae866aaea449f1e89564938bffd66fb3211147b4f3f", + "models/qwen4_exp/nvidia/mtp.py": "35dc7e4b4cf124ce79a7bfc65a5f198c9edc622f1692bf6b24c91486112b7d99", + "model_executor/layers/quantization/modelopt.py": "e65b339d573e1650508a9ab109c29f098e0a1cc799fbe04737e96596604fff02", + "config/engram.py": "34043a22385a041d4e9be8857f64174c990c2ca8308b97b6a952b5ab96c20ee5" +} diff --git a/audit/results.md b/audit/results.md new file mode 100644 index 0000000..60f1676 --- /dev/null +++ b/audit/results.md @@ -0,0 +1,41 @@ +# 实机验收：2026-09-18 + +机器：172.16.104.92，RTX 6000D 85651 MiB 显存、247 GiB 主机内存、16 vCPU、VMware。驱动 595.91.07。锁页内存 H2D 复制实测 57.81 GB/s，不能用虚拟 PCIe 链路显示值代替实测。 + +## 最终基线 + +128K（131072 token）、FP8 KV、FULL CUDA Graph capture size 1、单活动请求、2048 prefill batch、GPU memory utilization 0.96；保留视觉编码器；不开 MTP。PLE 日志确认 `weight_dtype=torch.float8_e4m3fn, weight_device=cpu, pinned=True`。 + +模型 GPU 加载约 73.77 GiB。Graph 配置 KV 预算约 3.27 GiB，日志容量 218453 token；这不改变服务设置的 131072 token 上限。CUDA Graph 捕获成功，实际捕获日志约 0.05 GiB。KV 容量和显存预算不能直接外推为多请求或更大视频安全容量。 + +## 正确性检查 + +- 官方 PLE 组件测试：24 passed，26 deselected（未运行 fused 子集）。 +- 官方 QSA sparse paged attention 数值测试：14 passed，58 deselected。 +- 11 个模型 safetensors SHA256 全部匹配，合计约 123.568 GiB。 +- 文字：中文固定回复、17×19；工具调用：get_weather/Shanghai，均通过。 +- 图片：红色合成图识别通过。 +- 视频：3 秒、9 帧红→绿→蓝视频识别通过。 +- 32K BF16 eager、128K FP8 eager、128K FP8 Graph 三阶段验收通过。 +- 128K Graph 检索实际输入 127988 token：首次首 token 14.141 秒、总计 14.887 秒；相同前缀复用首 token 0.454 秒、总计 1.214 秒。首 token 包含思考输出。 + +## CUDA Graph 对照 + +同一编程提示，temperature 0、seed 6000、reasoning_effort low，强制 256 个输出 token（含 212 个思考 token）。每组一次预热、三次测量，单请求，无 MTP。 + +| 模式 | 解码速度中位数 | 首 token 中位数 | +|---|---:|---:| +| eager | 22.002 token/s | 0.111 秒 | +| FULL Graph | 77.115 token/s | 0.070 秒 | + +约 **3.505 倍**。解码口径为 `(completion_tokens - 1) / (响应结束 - 首 token)`。固定长度用于公平计时，输出被截断，不作为该编程题质量评价。128K 首次预填充时间两组约 14.14 秒，Graph 提升主要出现在本次解码测试，不能解释为所有工作负载都提速 3.5 倍。 + +六次计时的正文和思考文本完全一致。最终启动后再次通过中文、数学、工具调用测试，内网健康接口 200、无鉴权模型接口 401，容器 healthy、自动重启策略 unless-stopped。最终监测 788 个采样中主机可用内存最低 172.743 GiB，未触发内存保护，主机 OOM kill 计数 0。 + +原始回执保存在 `runtime/`；服务器完整测试记录位于 `/data/flash-next/audit` 和 `/data/flash-next/logs`。 + +## 运维结论与边界 + +已修复缺失的 `nvidia-utils-595-server`，加载当前内核 NVIDIA 模块；CUDA 运算通过，无需为此次部署立即重启宿主机。已删除旧 MinerU/27B 相关容器、镜像和指定模型目录。 + +本次验收覆盖基本多模态、长输入检索、前缀复用和短时性能对照；没有完成长时间压力、并发请求或真实业务综合质量评估。默认不开 MTP，保留多模态及上下文预算。后续升级镜像、改变并发或启用 MTP 均需重新验收。 diff --git a/audit/runtime/acceptance-128k-fp8-eager.jsonl b/audit/runtime/acceptance-128k-fp8-eager.jsonl new file mode 100644 index 0000000..8df6d92 --- /dev/null +++ b/audit/runtime/acceptance-128k-fp8-eager.jsonl @@ -0,0 +1,4 @@ +{"models": {"object": "list", "data": [{"id": "qwen3.8-flash-next", "object": "model", "created": 1789698533, "owned_by": "vllm", "root": "/model", "parent": null, "max_model_len": 131072, "permission": [{"id": "modelperm-98ac33851ae14b2d", "object": "model_permission", "created": 1789698533, "allow_create_engine": false, "allow_sampling": true, "allow_logprobs": true, "allow_search_indices": false, "allow_view": true, "allow_fine_tuning": false, "organization": "*", "group": null, "is_blocking": false}]}]}} +{"test": "math", "passed": true, "content": "\n\n323", "reasoning_chars": 111, "ttft_s": 0.25864882499990927, "elapsed_s": 2.9955328199994256, "finish_reason": "stop", "usage": {"prompt_tokens": 55, "total_tokens": 115, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 54}}} +{"test": "chinese", "passed": true, "content": "\n\n模型已就绪", "reasoning_chars": 53, "ttft_s": 0.11319643099977839, "elapsed_s": 1.5984155480000481, "finish_reason": "stop", "usage": {"prompt_tokens": 47, "total_tokens": 80, "completion_tokens": 33, "completion_tokens_details": {"reasoning_tokens": 27}}} +{"test": "tool", "passed": true, "response": {"id": "chatcmpl-8cb69bf7f3e340ad", "object": "chat.completion", "created": 1789698537, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": null, "annotations": null, "audio": null, "function_call": null, "tool_calls": [{"id": "chatcmpl-tool-b61df6e1140dfffb", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Shanghai\"}"}}], "reasoning": "The user wants me to check the weather in Shanghai using the get_weather tool. This is a straightforward single tool call.\n"}, "logprobs": null, "finish_reason": "tool_calls", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-7d761c1a", "usage": {"prompt_tokens": 301, "total_tokens": 355, "completion_tokens": 54, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 25}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/acceptance-128k-graph.jsonl b/audit/runtime/acceptance-128k-graph.jsonl new file mode 100644 index 0000000..feec4d6 --- /dev/null +++ b/audit/runtime/acceptance-128k-graph.jsonl @@ -0,0 +1,4 @@ +{"models": {"object": "list", "data": [{"id": "qwen3.8-flash-next", "object": "model", "created": 1789698916, "owned_by": "vllm", "root": "/model", "parent": null, "max_model_len": 131072, "permission": [{"id": "modelperm-bd7a483e2f205e50", "object": "model_permission", "created": 1789698916, "allow_create_engine": false, "allow_sampling": true, "allow_logprobs": true, "allow_search_indices": false, "allow_view": true, "allow_fine_tuning": false, "organization": "*", "group": null, "is_blocking": false}]}]}} +{"test": "math", "passed": true, "content": "\n\n323", "reasoning_chars": 111, "ttft_s": 0.20529974200053402, "elapsed_s": 0.9841310700003305, "finish_reason": "stop", "usage": {"prompt_tokens": 55, "total_tokens": 115, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 54}}} +{"test": "chinese", "passed": true, "content": "\n\n模型已就绪", "reasoning_chars": 53, "ttft_s": 0.07831337100014935, "elapsed_s": 0.5017039430003933, "finish_reason": "stop", "usage": {"prompt_tokens": 47, "total_tokens": 80, "completion_tokens": 33, "completion_tokens_details": {"reasoning_tokens": 27}}} +{"test": "tool", "passed": true, "response": {"id": "chatcmpl-8032b15adb57ac50", "object": "chat.completion", "created": 1789698917, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": null, "annotations": null, "audio": null, "function_call": null, "tool_calls": [{"id": "chatcmpl-tool-bb4237a41958d7ec", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Shanghai\"}"}}], "reasoning": "The user wants me to check the weather in Shanghai using the get_weather tool. This is a straightforward single tool call.\n"}, "logprobs": null, "finish_reason": "tool_calls", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-9c4a3436", "usage": {"prompt_tokens": 301, "total_tokens": 355, "completion_tokens": 54, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 25}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/acceptance-32k.jsonl b/audit/runtime/acceptance-32k.jsonl new file mode 100644 index 0000000..996da03 --- /dev/null +++ b/audit/runtime/acceptance-32k.jsonl @@ -0,0 +1,4 @@ +{"models": {"object": "list", "data": [{"id": "qwen3.8-flash-next", "object": "model", "created": 1789698162, "owned_by": "vllm", "root": "/model", "parent": null, "max_model_len": 32768, "permission": [{"id": "modelperm-8dc326f840b3ca1d", "object": "model_permission", "created": 1789698162, "allow_create_engine": false, "allow_sampling": true, "allow_logprobs": true, "allow_search_indices": false, "allow_view": true, "allow_fine_tuning": false, "organization": "*", "group": null, "is_blocking": false}]}]}} +{"test": "math", "passed": true, "content": "\n\n323", "reasoning_chars": 111, "ttft_s": 0.2581440149997434, "elapsed_s": 3.016060242999629, "finish_reason": "stop", "usage": {"prompt_tokens": 55, "total_tokens": 115, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 54}}} +{"test": "chinese", "passed": true, "content": "\n\n模型已就绪", "reasoning_chars": 60, "ttft_s": 0.11625734200060833, "elapsed_s": 1.8704289390007034, "finish_reason": "stop", "usage": {"prompt_tokens": 47, "total_tokens": 85, "completion_tokens": 38, "completion_tokens_details": {"reasoning_tokens": 32}}} +{"test": "tool", "passed": true, "response": {"id": "chatcmpl-a39dd95107a5e097", "object": "chat.completion", "created": 1789698167, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": null, "annotations": null, "audio": null, "function_call": null, "tool_calls": [{"id": "chatcmpl-tool-b91e2ebd40c0203c", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Shanghai\"}"}}], "reasoning": "The user wants me to check the weather in Shanghai using the get_weather tool. This is a straightforward single call with no dependencies.\n"}, "logprobs": null, "finish_reason": "tool_calls", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-869b9ee8", "usage": {"prompt_tokens": 301, "total_tokens": 357, "completion_tokens": 56, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 27}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/acceptance-final-ready.jsonl b/audit/runtime/acceptance-final-ready.jsonl new file mode 100644 index 0000000..32c8ea0 --- /dev/null +++ b/audit/runtime/acceptance-final-ready.jsonl @@ -0,0 +1,4 @@ +{"models": {"object": "list", "data": [{"id": "qwen3.8-flash-next", "object": "model", "created": 1789699360, "owned_by": "vllm", "root": "/model", "parent": null, "max_model_len": 131072, "permission": [{"id": "modelperm-83769cdce59a78c8", "object": "model_permission", "created": 1789699360, "allow_create_engine": false, "allow_sampling": true, "allow_logprobs": true, "allow_search_indices": false, "allow_view": true, "allow_fine_tuning": false, "organization": "*", "group": null, "is_blocking": false}]}]}} +{"test": "math", "passed": true, "content": "\n\n323", "reasoning_chars": 111, "ttft_s": 0.2054933279996476, "elapsed_s": 0.9849798659997759, "finish_reason": "stop", "usage": {"prompt_tokens": 55, "total_tokens": 115, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 54}}} +{"test": "chinese", "passed": true, "content": "\n\n模型已就绪", "reasoning_chars": 53, "ttft_s": 0.07670184699964011, "elapsed_s": 0.5005528519996005, "finish_reason": "stop", "usage": {"prompt_tokens": 47, "total_tokens": 80, "completion_tokens": 33, "completion_tokens_details": {"reasoning_tokens": 27}}} +{"test": "tool", "passed": true, "response": {"id": "chatcmpl-b70f67f1f12703ce", "object": "chat.completion", "created": 1789699362, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": null, "annotations": null, "audio": null, "function_call": null, "tool_calls": [{"id": "chatcmpl-tool-a8ac09543d9dc917", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Shanghai\"}"}}], "reasoning": "The user wants me to check the weather in Shanghai using the get_weather tool. This is a straightforward single tool call.\n"}, "logprobs": null, "finish_reason": "tool_calls", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-9c4a3436", "usage": {"prompt_tokens": 301, "total_tokens": 355, "completion_tokens": 54, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 25}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/base-image.txt b/audit/runtime/base-image.txt new file mode 100644 index 0000000..d472973 --- /dev/null +++ b/audit/runtime/base-image.txt @@ -0,0 +1 @@ +sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28 amd64 diff --git a/audit/runtime/benchmark-eager.jsonl b/audit/runtime/benchmark-eager.jsonl new file mode 100644 index 0000000..e0681a9 --- /dev/null +++ b/audit/runtime/benchmark-eager.jsonl @@ -0,0 +1,4 @@ +{"run": 0, "warmup": true, "ttft_s": 0.12214227900039987, "elapsed_s": 11.675187773000289, "decode_tps": 22.072102125143978, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 1, "warmup": false, "ttft_s": 0.11141436400066596, "elapsed_s": 11.829587345000618, "decode_tps": 21.7610714924128, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 2, "warmup": false, "ttft_s": 0.11719201899995824, "elapsed_s": 11.707202442999915, "decode_tps": 22.001705837292434, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 3, "warmup": false, "ttft_s": 0.10968124400005763, "elapsed_s": 11.686878399000307, "decode_tps": 22.026056616809385, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} diff --git a/audit/runtime/benchmark-graph.jsonl b/audit/runtime/benchmark-graph.jsonl new file mode 100644 index 0000000..a28eb1e --- /dev/null +++ b/audit/runtime/benchmark-graph.jsonl @@ -0,0 +1,4 @@ +{"run": 0, "warmup": true, "ttft_s": 0.0754732100003821, "elapsed_s": 3.4145185560000755, "decode_tps": 76.36913356253156, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 1, "warmup": false, "ttft_s": 0.06956711800012272, "elapsed_s": 3.404837518000022, "decode_tps": 76.45557013908308, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 2, "warmup": false, "ttft_s": 0.06818878499962011, "elapsed_s": 3.374919455000054, "decode_tps": 77.11544284916513, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} +{"run": 3, "warmup": false, "ttft_s": 0.06954390600003535, "elapsed_s": 3.3748562399996445, "decode_tps": 77.14853370345065, "usage": {"prompt_tokens": 58, "total_tokens": 314, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 212}}, "content": "\n\n# Merging Overlapping Intervals\n\n## The Function\n\n```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\n \"\"\"\n", "reasoning": "The user wants a Python function that merges overlapping intervals, along with an explanation of the algorithm, edge cases, and complexity analysis.\n\nLet me think about the best approach:\n\n1. **Algorithm**: The classic approach is to sort intervals by start time, then iterate through them, merging when there's overlap. This is the most common and efficient approach.\n\n2. **Key considerations**:\n - What counts as \"overlapping\"? Adjacent intervals like [1,2] and [2,3] \u2014 should they merge? I'll handle both cases and mention it.\n - Edge cases: empty list, single interval, fully contained intervals, non-overlapping intervals, intervals touching at endpoints.\n\n3. **Complexity**: O(n log n) for sorting, O(n) for the merge pass, so O(n log n) overall. Space is O(n) for the output (or O(1) extra if we modify in place).\n\nLet me write a clean, well-documented solution with thorough explanation.\n"} diff --git a/audit/runtime/benchmark-summary.json b/audit/runtime/benchmark-summary.json new file mode 100644 index 0000000..c8a1ccb --- /dev/null +++ b/audit/runtime/benchmark-summary.json @@ -0,0 +1,12 @@ +{ + "eager": { + "median_decode_tps": 22.001705837292434, + "median_ttft_s": 0.11141436400066596 + }, + "graph": { + "median_decode_tps": 77.11544284916513, + "median_ttft_s": 0.06954390600003535 + }, + "speedup": 3.5049756332282227, + "identical_generated_text": true +} \ No newline at end of file diff --git a/audit/runtime/component-status b/audit/runtime/component-status new file mode 100644 index 0000000..b0aad4d --- /dev/null +++ b/audit/runtime/component-status @@ -0,0 +1 @@ +passed diff --git a/audit/runtime/compose-128k-fp8-eager.yaml b/audit/runtime/compose-128k-fp8-eager.yaml new file mode 100644 index 0000000..e195e4e --- /dev/null +++ b/audit/runtime/compose-128k-fp8-eager.yaml @@ -0,0 +1,45 @@ +name: qwen38-flash-rtx6000d +services: + vllm: + image: local/qwen38-flash-6000d:0bfc7a15 + build: . + container_name: qwen38-flash-6000d + gpus: all + shm_size: 8g + mem_limit: 210g + memswap_limit: 210g + restart: "no" + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000" + environment: + HF_HUB_OFFLINE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_WORKER_MULTIPROC_METHOD: spawn + OMP_NUM_THREADS: "8" + TORCHINDUCTOR_COMPILE_THREADS: "2" + CUTE_DSL_ARCH: sm_120a + volumes: + - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro + - /data/flash-next/cache:/root/.cache + - ./secrets/api-key:/run/secrets/api-key:ro + entrypoint: ["/bin/bash", "-lc"] + command: + - >- + exec vllm serve /model + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 --dtype bfloat16 + --engram-config '{"cpu_offload":true}' + --kv-cache-dtype fp8 --gpu-memory-utilization 0.96 + --max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048 + --enable-chunked-prefill --enable-prefix-caching + --enforce-eager --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/api-key)" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30m diff --git a/audit/runtime/compose-128k-graph.yaml b/audit/runtime/compose-128k-graph.yaml new file mode 100644 index 0000000..d49ef03 --- /dev/null +++ b/audit/runtime/compose-128k-graph.yaml @@ -0,0 +1,45 @@ +name: qwen38-flash-rtx6000d +services: + vllm: + image: local/qwen38-flash-6000d:0bfc7a15 + build: . + container_name: qwen38-flash-6000d + gpus: all + shm_size: 8g + mem_limit: 210g + memswap_limit: 210g + restart: "no" + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000" + environment: + HF_HUB_OFFLINE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_WORKER_MULTIPROC_METHOD: spawn + OMP_NUM_THREADS: "8" + TORCHINDUCTOR_COMPILE_THREADS: "2" + CUTE_DSL_ARCH: sm_120a + volumes: + - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro + - /data/flash-next/cache:/root/.cache + - ./secrets/api-key:/run/secrets/api-key:ro + entrypoint: ["/bin/bash", "-lc"] + command: + - >- + exec vllm serve /model + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 --dtype bfloat16 + --engram-config '{"cpu_offload":true}' + --kv-cache-dtype fp8 --gpu-memory-utilization 0.96 + --max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048 + --enable-chunked-prefill --enable-prefix-caching + --compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1]}' --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/api-key)" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30m diff --git a/audit/runtime/compose-32k-bf16.yaml b/audit/runtime/compose-32k-bf16.yaml new file mode 100644 index 0000000..85aee62 --- /dev/null +++ b/audit/runtime/compose-32k-bf16.yaml @@ -0,0 +1,45 @@ +name: qwen38-flash-rtx6000d +services: + vllm: + image: local/qwen38-flash-6000d:0bfc7a15 + build: . + container_name: qwen38-flash-6000d + gpus: all + shm_size: 8g + mem_limit: 210g + memswap_limit: 210g + restart: "no" + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000" + environment: + HF_HUB_OFFLINE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_WORKER_MULTIPROC_METHOD: spawn + OMP_NUM_THREADS: "8" + TORCHINDUCTOR_COMPILE_THREADS: "2" + CUTE_DSL_ARCH: sm_120a + volumes: + - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro + - /data/flash-next/cache:/root/.cache + - ./secrets/api-key:/run/secrets/api-key:ro + entrypoint: ["/bin/bash", "-lc"] + command: + - >- + exec vllm serve /model + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 --dtype bfloat16 + --engram-config '{"cpu_offload":true}' + --kv-cache-dtype auto --gpu-memory-utilization 0.94 + --max-model-len 32768 --max-num-seqs 1 --max-num-batched-tokens 2048 + --enable-chunked-prefill --enable-prefix-caching + --enforce-eager --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/api-key)" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30m diff --git a/audit/runtime/context-128k-fp8-eager.jsonl b/audit/runtime/context-128k-fp8-eager.jsonl new file mode 100644 index 0000000..a0e5ee6 --- /dev/null +++ b/audit/runtime/context-128k-fp8-eager.jsonl @@ -0,0 +1,2 @@ +{"repetition": 0, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 127988, "total_tokens": 128047, "completion_tokens": 59, "completion_tokens_details": {"reasoning_tokens": 47}}, "finish_reason": "stop", "ttft_s": 14.152911348999623, "elapsed_s": 16.802418123999814, "passed": true} +{"repetition": 1, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 127988, "total_tokens": 128047, "completion_tokens": 59, "completion_tokens_details": {"reasoning_tokens": 47}}, "finish_reason": "stop", "ttft_s": 0.4832818699997006, "elapsed_s": 3.1122699520001333, "passed": true} diff --git a/audit/runtime/context-128k-graph.jsonl b/audit/runtime/context-128k-graph.jsonl new file mode 100644 index 0000000..da68e4a --- /dev/null +++ b/audit/runtime/context-128k-graph.jsonl @@ -0,0 +1,2 @@ +{"repetition": 0, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 127988, "total_tokens": 128047, "completion_tokens": 59, "completion_tokens_details": {"reasoning_tokens": 47}}, "finish_reason": "stop", "ttft_s": 14.140956258999722, "elapsed_s": 14.887063680999745, "passed": true} +{"repetition": 1, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 127988, "total_tokens": 128048, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 48}}, "finish_reason": "stop", "ttft_s": 0.45431711799938057, "elapsed_s": 1.2140840899992327, "passed": true} diff --git a/audit/runtime/context-32k.jsonl b/audit/runtime/context-32k.jsonl new file mode 100644 index 0000000..5ea337f --- /dev/null +++ b/audit/runtime/context-32k.jsonl @@ -0,0 +1,2 @@ +{"repetition": 0, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 27982, "total_tokens": 28042, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 48}}, "finish_reason": "stop", "ttft_s": 3.2157685500005755, "elapsed_s": 5.955331851000665, "passed": true} +{"repetition": 1, "content": "\n\nVIOLET-74219", "usage": {"prompt_tokens": 27982, "total_tokens": 28042, "completion_tokens": 60, "completion_tokens_details": {"reasoning_tokens": 48}}, "finish_reason": "stop", "ttft_s": 0.222025799000221, "elapsed_s": 2.8937675820006916, "passed": true} diff --git a/audit/runtime/expected-weights.json b/audit/runtime/expected-weights.json new file mode 100644 index 0000000..c7c395f --- /dev/null +++ b/audit/runtime/expected-weights.json @@ -0,0 +1 @@ +{"model-00001-of-00010.safetensors": {"sha256": "63fde954be6f08b49b876f4f70a0ad0bcfee71aff7b1faa33779b6b32feca2a2", "bytes": 3115991696}, "model-00003-of-00010.safetensors": {"sha256": "3218ddc129258e91a721a8e81329ca8f00588332ec721bd8d2cf7d20e324bf47", "bytes": 10005364728}, "model-00004-of-00010.safetensors": {"sha256": "55e2bdf6a3a1f6e65270787f65c63b2a2a56b308b54eb1fb318dfabc9b48b141", "bytes": 10006060112}, "model-00007-of-00010.safetensors": {"sha256": "8cf3fa05c04cb2e060963b69bb01f7a7b9f43c58435fbe380836a0457a19cd3e", "bytes": 10005956544}, "model-00010-of-00010.safetensors": {"sha256": "0d49b0cf7c15bf9b5bd1e17403b36727315860a8f2a2e0f5a53655547084f640", "bytes": 128587536}, "model-00006-of-00010.safetensors": {"sha256": "38f65a9d11090428e139cc60d0587e8e5433ea2ce79820883c583d7cfcc53130", "bytes": 10005375328}, "model-fp8-mtp-ple.safetensors": {"sha256": "3525520c8602d850003eb1960aec0b64291dae33b83f8b65dd639a451df78823", "bytes": 53717551730}, "model-00005-of-00010.safetensors": {"sha256": "d3c169d3694bfba846455d01e48f047d770f78d071c5a4d22f129389411d686e", "bytes": 10005362392}, "model-00009-of-00010.safetensors": {"sha256": "eaff6a87ece6fbfd6ad0fbaa0a5cf9ff542f8076acba38e62410b1ed55e12cc3", "bytes": 5679113808}, "model-00002-of-00010.safetensors": {"sha256": "4dafaef62a908e49e0d92c7c2a3fa99f4d9651fdeeb0ca09f09a9b40095af4e0", "bytes": 10005510304}, "model-00008-of-00010.safetensors": {"sha256": "53d1f80746aa0cc0a7c2f4836587002a8a4dd72c827432cd2e43abae3c3899e4", "bytes": 10005375200}} diff --git a/audit/runtime/final-status.json b/audit/runtime/final-status.json new file mode 100644 index 0000000..51671ee --- /dev/null +++ b/audit/runtime/final-status.json @@ -0,0 +1,20 @@ +{ + "samples": 788, + "min_host_available_gib": 172.74323272705078, + "last_host_available_gib": 173.26424407958984, + "host_oom_kill_max": 0, + "safeguard_triggered": false, + "container_health": "healthy", + "container_oom_killed": false, + "restart_policy": "unless-stopped", + "ports": { + "8000/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "8000" + } + ] + }, + "lan_health": 200, + "unauth_models": 401 +} \ No newline at end of file diff --git a/audit/runtime/h2d.json b/audit/runtime/h2d.json new file mode 100644 index 0000000..e8ae665 --- /dev/null +++ b/audit/runtime/h2d.json @@ -0,0 +1 @@ +{"bytes_per_copy": 268435456, "copies": 20, "ms": 92.87158203125, "H2D_GB_s": 57.80787839054474, "correct": true} diff --git a/audit/runtime/multimodal-128k-fp8-eager.jsonl b/audit/runtime/multimodal-128k-fp8-eager.jsonl new file mode 100644 index 0000000..b22f426 --- /dev/null +++ b/audit/runtime/multimodal-128k-fp8-eager.jsonl @@ -0,0 +1,2 @@ +{"test": "image", "passed": true, "elapsed_s": 2.1940587230001256, "response": {"id": "chatcmpl-b6aef68d03f0613c", "object": "chat.completion", "created": 1789698540, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": "\n\nRed", "refusal": null, "annotations": null, "audio": null, "function_call": null, "reasoning": "The user wants the main color of the image in one English word. The image is a solid red field. The answer is simply \"Red\". I need to provide a concise, self-contained response.\n"}, "logprobs": null, "finish_reason": "stop", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-7d761c1a", "usage": {"prompt_tokens": 137, "total_tokens": 182, "completion_tokens": 45, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 41}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} +{"test": "video", "passed": true, "elapsed_s": 5.098227867999412, "response": {"id": "chatcmpl-a81f5755b276df9f", "object": "chat.completion", "created": 1789698542, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": "\n\nRed, Green, Blue", "refusal": null, "annotations": null, "audio": null, "function_call": null, "reasoning": "The user wants the background colors in chronological order from the three frames. Frame 1 (0.3s) is red, frame 2 (1.3s) is green, frame 3 (2.3s) is blue. I need to reply with only the three English color names. The instruction says \"Reply only with the three English color names.\" So I should provide just the names, likely comma-separated or space-separated. I'll provide \"Red, Green, Blue\".\n"}, "logprobs": null, "finish_reason": "stop", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-7d761c1a", "usage": {"prompt_tokens": 323, "total_tokens": 432, "completion_tokens": 109, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 101}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/multimodal-128k-graph.jsonl b/audit/runtime/multimodal-128k-graph.jsonl new file mode 100644 index 0000000..8d59aef --- /dev/null +++ b/audit/runtime/multimodal-128k-graph.jsonl @@ -0,0 +1,2 @@ +{"test": "image", "passed": true, "elapsed_s": 0.715891133999321, "response": {"id": "chatcmpl-9173d196000114e7", "object": "chat.completion", "created": 1789698918, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": "\n\nRed", "refusal": null, "annotations": null, "audio": null, "function_call": null, "reasoning": "The user wants the main color of the image in one English word. The image is a solid red field. The answer is simply \"Red\". I need to provide a concise, self-contained response.\n"}, "logprobs": null, "finish_reason": "stop", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-9c4a3436", "usage": {"prompt_tokens": 137, "total_tokens": 182, "completion_tokens": 45, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 41}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} +{"test": "video", "passed": true, "elapsed_s": 1.5345955559996582, "response": {"id": "chatcmpl-a178a2c9a19da528", "object": "chat.completion", "created": 1789698919, "model": "qwen3.8-flash-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": "\n\nRed, Green, Blue", "refusal": null, "annotations": null, "audio": null, "function_call": null, "reasoning": "The user wants the background colors in chronological order from the three frames. Frame 1 (0.3s) is red, frame 2 (1.3s) is green, frame 3 (2.3s) is blue. I need to reply with only the three English color names. The instruction says \"Reply only with the three English color names.\" So I should provide just the names, likely comma-separated or space-separated. I'll provide \"Red, Green, Blue\".\n"}, "logprobs": null, "finish_reason": "stop", "stop_reason": null, "token_ids": null, "routed_experts": null}], "service_tier": null, "system_fingerprint": "vllm-0.3.1.dev3+g0bfc7a15d-9c4a3436", "usage": {"prompt_tokens": 323, "total_tokens": 432, "completion_tokens": 109, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 101}}, "prompt_logprobs": null, "prompt_token_ids": null, "prompt_text": null, "kv_transfer_params": null, "ec_transfer_params": null, "metrics": null}} diff --git a/audit/runtime/old-containers.txt b/audit/runtime/old-containers.txt new file mode 100644 index 0000000..beac700 --- /dev/null +++ b/audit/runtime/old-containers.txt @@ -0,0 +1,3 @@ +flash-next-download docker.m.daocloud.io/vllm/vllm-openai:v0.27.1 Exited (1) 5 minutes ago +vllm-qwen38-27b-nvfp4-v0271 docker.m.daocloud.io/vllm/vllm-openai:v0.27.1 Up 4 minutes +mineru-3.4.5 local/mineru:3.4.5-fbb1257 Up 4 minutes (healthy) diff --git a/audit/runtime/ple-tests.log b/audit/runtime/ple-tests.log new file mode 100644 index 0000000..0da90a3 --- /dev/null +++ b/audit/runtime/ple-tests.log @@ -0,0 +1,12 @@ +........................ [100%] +=============================== warnings summary =============================== +../usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:365: 14 warnings + /usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +../usr/local/lib/python3.12/dist-packages/_pytest/cacheprovider.py:469 + /usr/local/lib/python3.12/dist-packages/_pytest/cacheprovider.py:469: PytestCacheWarning: could not create cache path /tests/.pytest_cache/v/cache/nodeids: [Errno 30] Read-only file system: '/tests/pytest-cache-files-1swv65lk' + config.cache.set("cache/nodeids", sorted(self.cached_nodeids)) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +24 passed, 26 deselected, 15 warnings in 14.78s diff --git a/audit/runtime/qsa-tests.log b/audit/runtime/qsa-tests.log new file mode 100644 index 0000000..17001b7 --- /dev/null +++ b/audit/runtime/qsa-tests.log @@ -0,0 +1,12 @@ +.............. [100%] +=============================== warnings summary =============================== +../usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:365: 14 warnings + /usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +../usr/local/lib/python3.12/dist-packages/_pytest/cacheprovider.py:469 + /usr/local/lib/python3.12/dist-packages/_pytest/cacheprovider.py:469: PytestCacheWarning: could not create cache path /tests/.pytest_cache/v/cache/nodeids: [Errno 30] Read-only file system: '/tests/pytest-cache-files-53g2vpwy' + config.cache.set("cache/nodeids", sorted(self.cached_nodeids)) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +14 passed, 58 deselected, 15 warnings in 25.10s diff --git a/audit/runtime/resources.jsonl b/audit/runtime/resources.jsonl new file mode 100644 index 0000000..fb3a343 --- /dev/null +++ b/audit/runtime/resources.jsonl @@ -0,0 +1,788 @@ +{"time": 1789697775.8689296, "available_gib": 242.79313278198242, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697777.89478, "available_gib": 242.7992172241211, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697779.9207852, "available_gib": 242.80086135864258, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697781.947063, "available_gib": 242.8029899597168, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697783.9729095, "available_gib": 242.80538177490234, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697785.9992328, "available_gib": 242.73623657226562, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697788.0240443, "available_gib": 242.61289596557617, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697790.0491273, "available_gib": 242.6235809326172, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697792.0758088, "available_gib": 242.62812423706055, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697794.1011505, "available_gib": 242.64812088012695, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697796.1274378, "available_gib": 242.65968322753906, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697798.1529548, "available_gib": 242.6789779663086, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697800.178232, "available_gib": 242.67703247070312, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697802.2036686, "available_gib": 242.69850158691406, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697804.2298043, "available_gib": 242.7092742919922, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697806.2553005, "available_gib": 242.72076797485352, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697808.2801383, "available_gib": 242.7169418334961, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697810.3056102, "available_gib": 242.7361831665039, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697812.3315756, "available_gib": 242.7453269958496, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697814.357228, "available_gib": 242.75972366333008, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697816.3826957, "available_gib": 242.77259063720703, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697818.4080167, "available_gib": 242.78713607788086, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697820.4347978, "available_gib": 242.79256057739258, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697822.4602606, "available_gib": 242.7982292175293, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697824.4858022, "available_gib": 242.7998504638672, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789697826.5114727, "available_gib": 242.8030891418457, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697828.5371664, "available_gib": 242.80643463134766, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697830.5635464, "available_gib": 242.81464385986328, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697832.589677, "available_gib": 242.81884384155273, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697834.6147816, "available_gib": 242.81423568725586, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697836.6421387, "available_gib": 242.8201560974121, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697838.6676874, "available_gib": 242.82007217407227, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697840.6928518, "available_gib": 242.81855010986328, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697842.7184668, "available_gib": 242.80772399902344, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697844.7444582, "available_gib": 242.81036376953125, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697846.7704813, "available_gib": 242.8054428100586, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697848.7961125, "available_gib": 242.8094024658203, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697850.8217428, "available_gib": 242.80743408203125, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697852.8473196, "available_gib": 242.80585098266602, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697854.8726041, "available_gib": 242.79705810546875, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697856.8977816, "available_gib": 242.79817962646484, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697858.924608, "available_gib": 242.81549835205078, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697860.950838, "available_gib": 242.81044387817383, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697862.9763281, "available_gib": 242.81206893920898, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697865.0021365, "available_gib": 242.80932998657227, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697867.0282185, "available_gib": 242.8117790222168, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697869.0546083, "available_gib": 242.81026458740234, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697871.0801868, "available_gib": 242.8094596862793, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697873.1063266, "available_gib": 242.80574417114258, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697875.1323574, "available_gib": 242.79839324951172, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697877.158424, "available_gib": 242.79821014404297, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697879.184704, "available_gib": 242.7967758178711, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697881.2106667, "available_gib": 242.79798889160156, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697883.2361698, "available_gib": 242.7956428527832, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697885.2617936, "available_gib": 242.79327774047852, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697887.2877278, "available_gib": 242.7908821105957, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697889.3136952, "available_gib": 242.7860107421875, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697891.3401275, "available_gib": 242.7905044555664, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697893.366173, "available_gib": 242.7957420349121, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697895.392468, "available_gib": 242.79530334472656, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697897.4185255, "available_gib": 242.7914161682129, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697899.4449077, "available_gib": 242.78981018066406, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697901.4712574, "available_gib": 242.79233932495117, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697903.4990776, "available_gib": 242.79348754882812, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697905.525157, "available_gib": 242.79033660888672, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697907.551171, "available_gib": 242.79165267944336, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697909.5773814, "available_gib": 242.78809356689453, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697911.6028702, "available_gib": 242.78575134277344, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697913.6290689, "available_gib": 242.7816505432129, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697915.6563382, "available_gib": 242.7847785949707, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697917.6820493, "available_gib": 242.7866973876953, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697919.708302, "available_gib": 242.78047561645508, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697921.7347534, "available_gib": 242.78277206420898, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697923.7610633, "available_gib": 242.78142547607422, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697925.787138, "available_gib": 242.77956008911133, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697927.8135931, "available_gib": 242.77875137329102, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697929.839674, "available_gib": 242.7775421142578, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697931.8654528, "available_gib": 242.78302764892578, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697933.8909419, "available_gib": 242.7862319946289, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697935.91726, "available_gib": 242.78515243530273, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697937.9432907, "available_gib": 242.77603149414062, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697939.9703052, "available_gib": 242.7667999267578, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697941.9956522, "available_gib": 242.77272033691406, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697944.0218155, "available_gib": 242.77366256713867, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697946.0473878, "available_gib": 242.77653121948242, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697948.0729365, "available_gib": 242.77483367919922, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697950.0991223, "available_gib": 242.77034759521484, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697952.1246293, "available_gib": 242.73120498657227, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697954.1505818, "available_gib": 242.72819137573242, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697956.1761153, "available_gib": 242.73687744140625, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697958.201761, "available_gib": 242.74141693115234, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697960.2273576, "available_gib": 242.74399948120117, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697962.2525682, "available_gib": 242.74022674560547, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697964.2782445, "available_gib": 242.74629974365234, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697966.3033783, "available_gib": 242.74235916137695, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697968.3288279, "available_gib": 242.7395896911621, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697970.3546672, "available_gib": 242.74538803100586, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697972.3803937, "available_gib": 242.74302673339844, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697974.406083, "available_gib": 242.74533462524414, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697976.431564, "available_gib": 242.74204635620117, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697978.4569016, "available_gib": 242.74512481689453, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697980.4823122, "available_gib": 242.74253845214844, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697982.507835, "available_gib": 242.7456932067871, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697984.5334477, "available_gib": 242.7455596923828, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697986.5588, "available_gib": 242.7530632019043, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697988.5839372, "available_gib": 242.75037384033203, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697990.6092942, "available_gib": 242.7528305053711, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697992.6337085, "available_gib": 242.7488250732422, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697994.658983, "available_gib": 242.74908828735352, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697996.6845617, "available_gib": 242.77647018432617, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789697998.71076, "available_gib": 242.77531051635742, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698000.737567, "available_gib": 242.77571868896484, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698002.7618816, "available_gib": 242.4694595336914, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698004.7866604, "available_gib": 242.3726921081543, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698006.8117993, "available_gib": 242.3055877685547, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698008.8358703, "available_gib": 242.2562599182129, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698010.8605874, "available_gib": 242.2424545288086, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698012.884564, "available_gib": 242.2011489868164, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698014.9088647, "available_gib": 242.11204528808594, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698016.9325619, "available_gib": 241.73583984375, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698018.9569397, "available_gib": 241.6239891052246, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698020.9807768, "available_gib": 241.50418090820312, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698023.0056794, "available_gib": 240.78117752075195, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698025.0376146, "available_gib": 241.85517120361328, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698027.0797858, "available_gib": 241.44847869873047, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698029.1221497, "available_gib": 241.21560668945312, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698031.1640868, "available_gib": 241.02487564086914, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698033.2069042, "available_gib": 240.6381721496582, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698035.2497404, "available_gib": 240.02870178222656, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698037.2919846, "available_gib": 239.83457946777344, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698039.336948, "available_gib": 239.32766342163086, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698041.3578022, "available_gib": 239.1604232788086, "oom_kill": 0, "gpu": "643, 85651, 0, 28", "stop": false} +{"time": 1789698043.3774905, "available_gib": 238.39139556884766, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698045.3977404, "available_gib": 234.04399490356445, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698047.4184196, "available_gib": 229.9498176574707, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698049.4385588, "available_gib": 226.01493072509766, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698051.4582741, "available_gib": 222.1092414855957, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698053.478506, "available_gib": 218.20440673828125, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698055.4987214, "available_gib": 214.1847267150879, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698057.518419, "available_gib": 210.20790481567383, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698059.5385299, "available_gib": 206.21024322509766, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698061.5581734, "available_gib": 202.226806640625, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698063.5782964, "available_gib": 198.19871139526367, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698065.599022, "available_gib": 194.05136489868164, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698067.6191502, "available_gib": 189.9062385559082, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698069.6394465, "available_gib": 186.03263092041016, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698071.6600935, "available_gib": 182.12273788452148, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698073.6798487, "available_gib": 179.12092208862305, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698075.6999524, "available_gib": 175.33677673339844, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698077.7216837, "available_gib": 174.63466262817383, "oom_kill": 0, "gpu": "76915, 85651, 3, 31", "stop": false} +{"time": 1789698079.7422805, "available_gib": 174.66100311279297, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698081.7634487, "available_gib": 174.64849090576172, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698083.786106, "available_gib": 174.6141471862793, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698085.8083165, "available_gib": 174.60975646972656, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698087.830143, "available_gib": 174.60482025146484, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789698089.8529222, "available_gib": 174.58005142211914, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789698091.874257, "available_gib": 174.5653305053711, "oom_kill": 0, "gpu": "76915, 85651, 8, 31", "stop": false} +{"time": 1789698093.8948672, "available_gib": 174.55259323120117, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698095.9157314, "available_gib": 174.55759811401367, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698097.9364295, "available_gib": 174.5730857849121, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698099.956956, "available_gib": 174.5580062866211, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698101.9776165, "available_gib": 174.55781936645508, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698103.9983711, "available_gib": 174.54605102539062, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789698106.0189106, "available_gib": 174.54510498046875, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698108.039702, "available_gib": 174.54458618164062, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698110.0606499, "available_gib": 174.54230499267578, "oom_kill": 0, "gpu": "76915, 85651, 5, 31", "stop": false} +{"time": 1789698112.0813212, "available_gib": 174.54075241088867, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698114.1024182, "available_gib": 174.53675842285156, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698116.1237397, "available_gib": 174.53759002685547, "oom_kill": 0, "gpu": "76915, 85651, 8, 32", "stop": false} +{"time": 1789698118.1449742, "available_gib": 174.5290069580078, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698120.1684828, "available_gib": 174.49939346313477, "oom_kill": 0, "gpu": "76915, 85651, 0, 32", "stop": false} +{"time": 1789698122.1878958, "available_gib": 174.3678436279297, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698124.208516, "available_gib": 174.01903915405273, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698126.2281425, "available_gib": 173.21017837524414, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698128.2485006, "available_gib": 173.11386489868164, "oom_kill": 0, "gpu": "78391, 85651, 0, 32", "stop": false} +{"time": 1789698130.2684126, "available_gib": 173.09959411621094, "oom_kill": 0, "gpu": "78391, 85651, 0, 32", "stop": false} +{"time": 1789698132.2881312, "available_gib": 173.07294845581055, "oom_kill": 0, "gpu": "78391, 85651, 0, 32", "stop": false} +{"time": 1789698134.3093653, "available_gib": 173.0535888671875, "oom_kill": 0, "gpu": "79611, 85651, 100, 42", "stop": false} +{"time": 1789698136.3304417, "available_gib": 173.0723991394043, "oom_kill": 0, "gpu": "79611, 85651, 100, 43", "stop": false} +{"time": 1789698138.3515165, "available_gib": 173.0891990661621, "oom_kill": 0, "gpu": "79613, 85651, 0, 36", "stop": false} +{"time": 1789698140.372323, "available_gib": 173.1020050048828, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698142.393099, "available_gib": 173.12941360473633, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698144.4138331, "available_gib": 173.1462631225586, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698146.4359224, "available_gib": 173.15244674682617, "oom_kill": 0, "gpu": "76979, 85651, 0, 35", "stop": false} +{"time": 1789698148.4569354, "available_gib": 173.29345321655273, "oom_kill": 0, "gpu": "78467, 85651, 0, 34", "stop": false} +{"time": 1789698150.476658, "available_gib": 173.25193405151367, "oom_kill": 0, "gpu": "78467, 85651, 0, 33", "stop": false} +{"time": 1789698152.4977326, "available_gib": 173.2181053161621, "oom_kill": 0, "gpu": "78467, 85651, 1, 33", "stop": false} +{"time": 1789698154.518783, "available_gib": 173.13046264648438, "oom_kill": 0, "gpu": "78501, 85651, 0, 33", "stop": false} +{"time": 1789698156.5389428, "available_gib": 173.01626586914062, "oom_kill": 0, "gpu": "78509, 85651, 0, 33", "stop": false} +{"time": 1789698158.560792, "available_gib": 172.9888458251953, "oom_kill": 0, "gpu": "78509, 85651, 0, 33", "stop": false} +{"time": 1789698160.5806274, "available_gib": 172.9702911376953, "oom_kill": 0, "gpu": "78509, 85651, 0, 33", "stop": false} +{"time": 1789698162.6029568, "available_gib": 172.94962692260742, "oom_kill": 0, "gpu": "78509, 85651, 43, 35", "stop": false} +{"time": 1789698164.6251001, "available_gib": 172.94060134887695, "oom_kill": 0, "gpu": "78509, 85651, 34, 35", "stop": false} +{"time": 1789698166.6472292, "available_gib": 172.94208908081055, "oom_kill": 0, "gpu": "78511, 85651, 33, 35", "stop": false} +{"time": 1789698168.6694615, "available_gib": 172.95393753051758, "oom_kill": 0, "gpu": "78511, 85651, 33, 35", "stop": false} +{"time": 1789698170.6932688, "available_gib": 172.9192771911621, "oom_kill": 0, "gpu": "78681, 85651, 100, 44", "stop": false} +{"time": 1789698172.7191985, "available_gib": 172.9352569580078, "oom_kill": 0, "gpu": "78681, 85651, 100, 45", "stop": false} +{"time": 1789698174.7422264, "available_gib": 172.94940185546875, "oom_kill": 0, "gpu": "78681, 85651, 34, 38", "stop": false} +{"time": 1789698176.7653575, "available_gib": 172.96106719970703, "oom_kill": 0, "gpu": "78681, 85651, 34, 38", "stop": false} +{"time": 1789698178.788159, "available_gib": 172.97318649291992, "oom_kill": 0, "gpu": "78681, 85651, 35, 37", "stop": false} +{"time": 1789698180.8089764, "available_gib": 172.9854507446289, "oom_kill": 0, "gpu": "78681, 85651, 0, 36", "stop": false} +{"time": 1789698182.8288891, "available_gib": 172.9985809326172, "oom_kill": 0, "gpu": "78681, 85651, 0, 35", "stop": false} +{"time": 1789698184.8487601, "available_gib": 173.00785446166992, "oom_kill": 0, "gpu": "78681, 85651, 0, 35", "stop": false} +{"time": 1789698186.8683662, "available_gib": 172.99964904785156, "oom_kill": 0, "gpu": "78681, 85651, 0, 35", "stop": false} +{"time": 1789698188.8882835, "available_gib": 172.99706268310547, "oom_kill": 0, "gpu": "78681, 85651, 0, 35", "stop": false} +{"time": 1789698190.9081326, "available_gib": 172.99339294433594, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698192.9278023, "available_gib": 172.9930877685547, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698194.9482439, "available_gib": 172.98613357543945, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698196.9682136, "available_gib": 172.99684524536133, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698198.9883125, "available_gib": 172.99680709838867, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698201.008116, "available_gib": 172.99489974975586, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698203.0330844, "available_gib": 172.99774551391602, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698205.05835, "available_gib": 173.00514221191406, "oom_kill": 0, "gpu": "78681, 85651, 0, 34", "stop": false} +{"time": 1789698207.0841086, "available_gib": 173.0086898803711, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698209.1038861, "available_gib": 172.99974060058594, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698211.1237545, "available_gib": 172.99799346923828, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698213.143623, "available_gib": 172.99457168579102, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698215.1633854, "available_gib": 172.99704360961914, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698217.183249, "available_gib": 173.00056076049805, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698219.2031782, "available_gib": 173.0000343322754, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698221.2228048, "available_gib": 172.99780654907227, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698223.2424433, "available_gib": 172.9866600036621, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698225.2619169, "available_gib": 172.97312545776367, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698227.2814896, "available_gib": 172.9788360595703, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698229.3010585, "available_gib": 172.97061157226562, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698231.320651, "available_gib": 172.969482421875, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698233.3406456, "available_gib": 172.96633911132812, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698235.3602703, "available_gib": 172.97027206420898, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698237.3796523, "available_gib": 172.9760398864746, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698239.3992674, "available_gib": 172.95990753173828, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698241.4190645, "available_gib": 172.96183013916016, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698243.438755, "available_gib": 172.96820068359375, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698245.4583094, "available_gib": 172.96745681762695, "oom_kill": 0, "gpu": "78681, 85651, 0, 33", "stop": false} +{"time": 1789698247.47804, "available_gib": 172.9650650024414, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698249.497772, "available_gib": 172.96280670166016, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698251.5175037, "available_gib": 172.96283340454102, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698253.5375342, "available_gib": 172.96292114257812, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698255.557578, "available_gib": 172.9600601196289, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698257.5774462, "available_gib": 172.96017837524414, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698259.5972764, "available_gib": 172.9626007080078, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698261.6169975, "available_gib": 172.95975494384766, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698263.6368196, "available_gib": 172.95936965942383, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698265.6564972, "available_gib": 172.95799255371094, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698267.6764672, "available_gib": 172.95737075805664, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698269.6962109, "available_gib": 172.96297454833984, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698271.716137, "available_gib": 172.9764404296875, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698273.735803, "available_gib": 172.97857284545898, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698275.755253, "available_gib": 172.96168518066406, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698277.7752292, "available_gib": 172.95572662353516, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698279.7950063, "available_gib": 172.96673583984375, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698281.8151293, "available_gib": 172.97986602783203, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698283.835362, "available_gib": 172.99542236328125, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698285.8559701, "available_gib": 172.9915542602539, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698287.8765326, "available_gib": 172.9981460571289, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698289.8967414, "available_gib": 173.00347900390625, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698291.91676, "available_gib": 173.01255416870117, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698293.936605, "available_gib": 173.01031112670898, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698295.9572845, "available_gib": 174.69717025756836, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698297.9779153, "available_gib": 175.47014999389648, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698299.9990237, "available_gib": 175.46588134765625, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698302.0197637, "available_gib": 175.4655876159668, "oom_kill": 0, "gpu": "78681, 85651, 0, 32", "stop": false} +{"time": 1789698310.5263555, "available_gib": 175.98656463623047, "oom_kill": 0, "gpu": "3, 85651, 0, 32", "stop": false} +{"time": 1789698312.5487058, "available_gib": 242.54931259155273, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698314.570991, "available_gib": 242.4609718322754, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698316.5949636, "available_gib": 242.3786163330078, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698318.6198661, "available_gib": 242.33044052124023, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789698320.6447208, "available_gib": 242.31953811645508, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789698322.6702259, "available_gib": 242.26551055908203, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789698324.694822, "available_gib": 242.1980209350586, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789698326.7219899, "available_gib": 242.65842056274414, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789698328.746445, "available_gib": 242.8007698059082, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698330.7720232, "available_gib": 242.80065155029297, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698332.7978075, "available_gib": 242.79240798950195, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698334.823254, "available_gib": 242.79387283325195, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698336.8489504, "available_gib": 242.7922019958496, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698338.874753, "available_gib": 242.79584121704102, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698340.9001236, "available_gib": 242.78631973266602, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698342.925491, "available_gib": 242.7849349975586, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789698344.9509962, "available_gib": 242.78447723388672, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698346.9758806, "available_gib": 242.7764434814453, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698349.00125, "available_gib": 242.7763786315918, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698351.026546, "available_gib": 242.7745590209961, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698353.0519142, "available_gib": 242.77412033081055, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698355.077221, "available_gib": 242.7719841003418, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698357.10255, "available_gib": 242.78934478759766, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698359.127974, "available_gib": 242.7824363708496, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698361.153463, "available_gib": 242.78175735473633, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698363.1792793, "available_gib": 242.78052520751953, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698365.2045178, "available_gib": 242.7781219482422, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698367.229652, "available_gib": 242.78032684326172, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698369.2553208, "available_gib": 242.77717208862305, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698371.2807386, "available_gib": 242.7810516357422, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698373.306037, "available_gib": 242.78021240234375, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698375.331409, "available_gib": 242.7786407470703, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698377.3566995, "available_gib": 242.77531814575195, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698379.3820624, "available_gib": 242.7735481262207, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698381.4072747, "available_gib": 242.7775001525879, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false} +{"time": 1789698383.4316342, "available_gib": 242.53756713867188, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698385.4555943, "available_gib": 242.44918060302734, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698387.4800208, "available_gib": 242.35544204711914, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698389.5044615, "available_gib": 242.31780624389648, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698391.5287435, "available_gib": 242.28925704956055, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698393.5525866, "available_gib": 242.244873046875, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698395.5768516, "available_gib": 242.17669296264648, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698397.6013536, "available_gib": 242.12627029418945, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false} +{"time": 1789698399.6255953, "available_gib": 241.7772979736328, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698401.668438, "available_gib": 241.50298690795898, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698403.7106304, "available_gib": 241.1844825744629, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698405.7555416, "available_gib": 241.02505111694336, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698407.7985668, "available_gib": 240.4225616455078, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698409.8403904, "available_gib": 240.09104919433594, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698411.882191, "available_gib": 239.78043746948242, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698413.9240148, "available_gib": 239.42470169067383, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false} +{"time": 1789698415.9437115, "available_gib": 239.1597785949707, "oom_kill": 0, "gpu": "2873, 85651, 0, 29", "stop": false} +{"time": 1789698417.9638798, "available_gib": 235.23350524902344, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698419.9834926, "available_gib": 230.90675735473633, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698422.0030177, "available_gib": 226.85679626464844, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698424.022438, "available_gib": 222.83197021484375, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698426.0419853, "available_gib": 218.80087661743164, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698428.0614889, "available_gib": 214.796142578125, "oom_kill": 0, "gpu": "4245, 85651, 0, 29", "stop": false} +{"time": 1789698430.081392, "available_gib": 210.85106658935547, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698432.1009026, "available_gib": 206.8211669921875, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698434.12081, "available_gib": 202.82791900634766, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698436.1406634, "available_gib": 198.8251075744629, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698438.1599734, "available_gib": 194.85071563720703, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698440.180275, "available_gib": 190.83799743652344, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698442.200231, "available_gib": 186.83391571044922, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698444.220073, "available_gib": 182.84481048583984, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698446.2394726, "available_gib": 178.82078552246094, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698448.259143, "available_gib": 174.8134880065918, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789698450.2800634, "available_gib": 173.24686431884766, "oom_kill": 0, "gpu": "72653, 85651, 4, 31", "stop": false} +{"time": 1789698452.303277, "available_gib": 173.26063919067383, "oom_kill": 0, "gpu": "76915, 85651, 3, 31", "stop": false} +{"time": 1789698454.3237903, "available_gib": 173.48651504516602, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698456.3444817, "available_gib": 173.52472686767578, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698458.3653955, "available_gib": 173.53572845458984, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698460.386637, "available_gib": 173.56494140625, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698462.4080014, "available_gib": 173.59941482543945, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698464.4289658, "available_gib": 173.58258438110352, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698466.4498763, "available_gib": 173.6162338256836, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698468.4706445, "available_gib": 173.66310501098633, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698470.4915016, "available_gib": 173.6963996887207, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698472.5126371, "available_gib": 173.70503616333008, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698474.5334997, "available_gib": 173.76955795288086, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698476.554281, "available_gib": 173.7882308959961, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698478.5749776, "available_gib": 173.8198127746582, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698480.5963385, "available_gib": 173.8772201538086, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698482.6178179, "available_gib": 173.89176177978516, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698484.639224, "available_gib": 173.91453552246094, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698486.660508, "available_gib": 173.95732879638672, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789698488.6814284, "available_gib": 173.97729873657227, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698490.7024295, "available_gib": 173.9884033203125, "oom_kill": 0, "gpu": "76915, 85651, 26, 31", "stop": false} +{"time": 1789698492.72438, "available_gib": 173.95774459838867, "oom_kill": 0, "gpu": "76915, 85651, 0, 31", "stop": false} +{"time": 1789698494.7451446, "available_gib": 173.75876998901367, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698496.765553, "available_gib": 173.46038436889648, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698498.7860065, "available_gib": 172.8456268310547, "oom_kill": 0, "gpu": "76949, 85651, 0, 32", "stop": false} +{"time": 1789698500.8064108, "available_gib": 172.79693222045898, "oom_kill": 0, "gpu": "78391, 85651, 0, 31", "stop": false} +{"time": 1789698502.8261619, "available_gib": 172.79009628295898, "oom_kill": 0, "gpu": "78391, 85651, 0, 31", "stop": false} +{"time": 1789698504.8485017, "available_gib": 172.79087829589844, "oom_kill": 0, "gpu": "79611, 85651, 100, 38", "stop": false} +{"time": 1789698506.8706155, "available_gib": 172.80596923828125, "oom_kill": 0, "gpu": "79611, 85651, 100, 42", "stop": false} +{"time": 1789698508.8924623, "available_gib": 172.8499298095703, "oom_kill": 0, "gpu": "79611, 85651, 100, 43", "stop": false} +{"time": 1789698510.913065, "available_gib": 172.87175750732422, "oom_kill": 0, "gpu": "79615, 85651, 0, 35", "stop": false} +{"time": 1789698512.933405, "available_gib": 172.8903923034668, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698514.9538093, "available_gib": 172.9257469177246, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698516.973653, "available_gib": 172.8691062927246, "oom_kill": 0, "gpu": "79615, 85651, 0, 33", "stop": false} +{"time": 1789698518.993959, "available_gib": 173.10041427612305, "oom_kill": 0, "gpu": "76979, 85651, 0, 34", "stop": false} +{"time": 1789698521.0141978, "available_gib": 173.08990478515625, "oom_kill": 0, "gpu": "80199, 85651, 0, 33", "stop": false} +{"time": 1789698523.0339067, "available_gib": 173.05687713623047, "oom_kill": 0, "gpu": "80199, 85651, 0, 33", "stop": false} +{"time": 1789698525.0538027, "available_gib": 173.0294075012207, "oom_kill": 0, "gpu": "80199, 85651, 0, 33", "stop": false} +{"time": 1789698527.0746229, "available_gib": 173.00941467285156, "oom_kill": 0, "gpu": "80199, 85651, 0, 33", "stop": false} +{"time": 1789698529.095018, "available_gib": 172.9038963317871, "oom_kill": 0, "gpu": "80233, 85651, 0, 33", "stop": false} +{"time": 1789698531.1155448, "available_gib": 172.76727294921875, "oom_kill": 0, "gpu": "80241, 85651, 0, 33", "stop": false} +{"time": 1789698533.1355271, "available_gib": 172.76806640625, "oom_kill": 0, "gpu": "80241, 85651, 0, 33", "stop": false} +{"time": 1789698535.1581118, "available_gib": 172.7804946899414, "oom_kill": 0, "gpu": "80241, 85651, 34, 35", "stop": false} +{"time": 1789698537.1804023, "available_gib": 172.80339813232422, "oom_kill": 0, "gpu": "80243, 85651, 34, 35", "stop": false} +{"time": 1789698539.202662, "available_gib": 172.8612060546875, "oom_kill": 0, "gpu": "80243, 85651, 34, 35", "stop": false} +{"time": 1789698541.225123, "available_gib": 172.84925079345703, "oom_kill": 0, "gpu": "80243, 85651, 35, 35", "stop": false} +{"time": 1789698543.24739, "available_gib": 172.84303283691406, "oom_kill": 0, "gpu": "80243, 85651, 34, 35", "stop": false} +{"time": 1789698545.2698822, "available_gib": 172.87358856201172, "oom_kill": 0, "gpu": "80243, 85651, 34, 35", "stop": false} +{"time": 1789698547.291919, "available_gib": 172.92445755004883, "oom_kill": 0, "gpu": "80243, 85651, 34, 36", "stop": false} +{"time": 1789698549.3128505, "available_gib": 172.8004493713379, "oom_kill": 0, "gpu": "80243, 85651, 0, 34", "stop": false} +{"time": 1789698551.3388336, "available_gib": 172.78339004516602, "oom_kill": 0, "gpu": "80481, 85651, 100, 45", "stop": false} +{"time": 1789698553.3625271, "available_gib": 172.775634765625, "oom_kill": 0, "gpu": "80481, 85651, 100, 46", "stop": false} +{"time": 1789698555.3850305, "available_gib": 172.75973892211914, "oom_kill": 0, "gpu": "80481, 85651, 100, 46", "stop": false} +{"time": 1789698557.4091973, "available_gib": 172.7776756286621, "oom_kill": 0, "gpu": "80481, 85651, 100, 48", "stop": false} +{"time": 1789698559.4326754, "available_gib": 172.76174926757812, "oom_kill": 0, "gpu": "80481, 85651, 100, 48", "stop": false} +{"time": 1789698561.4560819, "available_gib": 172.75803756713867, "oom_kill": 0, "gpu": "80901, 85651, 100, 49", "stop": false} +{"time": 1789698563.480898, "available_gib": 172.80412673950195, "oom_kill": 0, "gpu": "82091, 85651, 100, 50", "stop": false} +{"time": 1789698565.5028596, "available_gib": 172.7872772216797, "oom_kill": 0, "gpu": "82091, 85651, 35, 42", "stop": false} +{"time": 1789698567.525117, "available_gib": 172.78892135620117, "oom_kill": 0, "gpu": "82091, 85651, 35, 42", "stop": false} +{"time": 1789698569.5456069, "available_gib": 172.83770751953125, "oom_kill": 0, "gpu": "82091, 85651, 0, 40", "stop": false} +{"time": 1789698571.5659344, "available_gib": 172.84392166137695, "oom_kill": 0, "gpu": "82091, 85651, 0, 39", "stop": false} +{"time": 1789698573.5857096, "available_gib": 172.85082626342773, "oom_kill": 0, "gpu": "82091, 85651, 0, 39", "stop": false} +{"time": 1789698575.6056898, "available_gib": 172.8469009399414, "oom_kill": 0, "gpu": "82091, 85651, 0, 38", "stop": false} +{"time": 1789698577.6256294, "available_gib": 172.8496437072754, "oom_kill": 0, "gpu": "82091, 85651, 0, 38", "stop": false} +{"time": 1789698579.6456003, "available_gib": 172.85250854492188, "oom_kill": 0, "gpu": "82091, 85651, 0, 38", "stop": false} +{"time": 1789698581.6655452, "available_gib": 172.84910583496094, "oom_kill": 0, "gpu": "82091, 85651, 0, 37", "stop": false} +{"time": 1789698583.685527, "available_gib": 172.8551788330078, "oom_kill": 0, "gpu": "82091, 85651, 0, 37", "stop": false} +{"time": 1789698585.705676, "available_gib": 172.85332107543945, "oom_kill": 0, "gpu": "82091, 85651, 0, 37", "stop": false} +{"time": 1789698587.7258303, "available_gib": 172.85873413085938, "oom_kill": 0, "gpu": "82091, 85651, 0, 36", "stop": false} +{"time": 1789698589.7462752, "available_gib": 172.8616600036621, "oom_kill": 0, "gpu": "82091, 85651, 0, 36", "stop": false} +{"time": 1789698591.7669246, "available_gib": 172.86176681518555, "oom_kill": 0, "gpu": "82091, 85651, 0, 36", "stop": false} +{"time": 1789698593.78716, "available_gib": 172.8709259033203, "oom_kill": 0, "gpu": "82091, 85651, 0, 36", "stop": false} +{"time": 1789698595.8071263, "available_gib": 172.86662673950195, "oom_kill": 0, "gpu": "82091, 85651, 0, 36", "stop": false} +{"time": 1789698597.8271146, "available_gib": 172.87282180786133, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698599.8469148, "available_gib": 172.86485290527344, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698601.8668633, "available_gib": 172.86913681030273, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698603.8868704, "available_gib": 172.87607192993164, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698605.907193, "available_gib": 172.8852882385254, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698607.9272501, "available_gib": 172.88738632202148, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698609.9473422, "available_gib": 172.88732147216797, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698611.967142, "available_gib": 172.88496017456055, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698613.9872987, "available_gib": 172.88508987426758, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698616.0072973, "available_gib": 172.88427734375, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698618.0272708, "available_gib": 172.88373184204102, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698620.047239, "available_gib": 172.87247848510742, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698622.067221, "available_gib": 172.88274383544922, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698624.0871544, "available_gib": 172.87973022460938, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698626.1069355, "available_gib": 172.873291015625, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698628.1267688, "available_gib": 172.87873077392578, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698630.1465788, "available_gib": 172.8780975341797, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698632.1663058, "available_gib": 172.87694549560547, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698634.1862938, "available_gib": 172.87271118164062, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698636.2061486, "available_gib": 172.88341522216797, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698638.226158, "available_gib": 172.87541961669922, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698640.2461534, "available_gib": 172.87984085083008, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698642.2659996, "available_gib": 172.87789916992188, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698644.2857196, "available_gib": 172.87405395507812, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698646.3055127, "available_gib": 172.8747444152832, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698648.3253098, "available_gib": 172.8754539489746, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698650.3453736, "available_gib": 172.87712860107422, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698652.365169, "available_gib": 172.87312698364258, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698654.3850434, "available_gib": 172.87524795532227, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698656.4046695, "available_gib": 172.87042236328125, "oom_kill": 0, "gpu": "82091, 85651, 0, 33", "stop": false} +{"time": 1789698658.424345, "available_gib": 172.8726577758789, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698660.4442298, "available_gib": 172.86931228637695, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698662.463871, "available_gib": 172.8721466064453, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698664.483628, "available_gib": 172.8698844909668, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698666.5033624, "available_gib": 172.8728256225586, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698668.523286, "available_gib": 172.86841583251953, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698670.5433393, "available_gib": 172.87340927124023, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698672.5624459, "available_gib": 172.81938934326172, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698674.5825434, "available_gib": 172.82315063476562, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698676.6028075, "available_gib": 172.83884048461914, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698678.6228917, "available_gib": 172.84679794311523, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698680.643129, "available_gib": 172.84695053100586, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698682.6632624, "available_gib": 172.85445404052734, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698684.6836066, "available_gib": 172.84949493408203, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698686.7035723, "available_gib": 172.85165405273438, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698688.7236552, "available_gib": 172.8495330810547, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698690.7441537, "available_gib": 172.85557556152344, "oom_kill": 0, "gpu": "82091, 85651, 0, 32", "stop": false} +{"time": 1789698692.7666724, "available_gib": 172.84199142456055, "oom_kill": 0, "gpu": "82091, 85651, 34, 34", "stop": false} +{"time": 1789698694.7896402, "available_gib": 172.84075164794922, "oom_kill": 0, "gpu": "82091, 85651, 34, 34", "stop": false} +{"time": 1789698696.812245, "available_gib": 172.85233688354492, "oom_kill": 0, "gpu": "82091, 85651, 34, 34", "stop": false} +{"time": 1789698698.8344452, "available_gib": 172.85191345214844, "oom_kill": 0, "gpu": "82091, 85651, 34, 34", "stop": false} +{"time": 1789698700.8568234, "available_gib": 172.8371696472168, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698702.879144, "available_gib": 172.84121704101562, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698704.9017868, "available_gib": 172.8515625, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698706.9244428, "available_gib": 172.84825134277344, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698708.947143, "available_gib": 172.8551483154297, "oom_kill": 0, "gpu": "82091, 85651, 33, 35", "stop": false} +{"time": 1789698710.9695776, "available_gib": 172.85575103759766, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698712.992177, "available_gib": 172.85752868652344, "oom_kill": 0, "gpu": "82091, 85651, 34, 35", "stop": false} +{"time": 1789698715.014442, "available_gib": 172.8544158935547, "oom_kill": 0, "gpu": "82091, 85651, 33, 36", "stop": false} +{"time": 1789698717.0366125, "available_gib": 172.83416366577148, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698719.0588489, "available_gib": 172.81727600097656, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698721.0815356, "available_gib": 172.81825256347656, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698723.1040015, "available_gib": 172.8337059020996, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698725.1263435, "available_gib": 172.83274459838867, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698727.1489973, "available_gib": 172.8340301513672, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698729.1716938, "available_gib": 172.8365364074707, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698731.194568, "available_gib": 172.83840560913086, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698733.2170594, "available_gib": 172.8403091430664, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698735.2394729, "available_gib": 172.84373092651367, "oom_kill": 0, "gpu": "82091, 85651, 34, 36", "stop": false} +{"time": 1789698737.2621658, "available_gib": 172.84193420410156, "oom_kill": 0, "gpu": "82091, 85651, 34, 37", "stop": false} +{"time": 1789698739.2830215, "available_gib": 174.5977554321289, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698741.303102, "available_gib": 174.60450744628906, "oom_kill": 0, "gpu": "82091, 85651, 0, 35", "stop": false} +{"time": 1789698743.32384, "available_gib": 175.37614822387695, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698745.3443983, "available_gib": 175.38105392456055, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698747.3650422, "available_gib": 175.37316513061523, "oom_kill": 0, "gpu": "82091, 85651, 0, 34", "stop": false} +{"time": 1789698755.1502216, "available_gib": 181.98407745361328, "oom_kill": 0, "gpu": "3, 85651, 0, 34", "stop": false} +{"time": 1789698757.1732614, "available_gib": 242.48251724243164, "oom_kill": 0, "gpu": "0, 85651, 0, 34", "stop": false} +{"time": 1789698759.1951036, "available_gib": 242.39908981323242, "oom_kill": 0, "gpu": "0, 85651, 0, 32", "stop": false} +{"time": 1789698761.2198927, "available_gib": 242.31695938110352, "oom_kill": 0, "gpu": "0, 85651, 0, 32", "stop": false} +{"time": 1789698763.2441344, "available_gib": 242.2816925048828, "oom_kill": 0, "gpu": "0, 85651, 0, 32", "stop": false} +{"time": 1789698765.268554, "available_gib": 242.25524520874023, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698767.293194, "available_gib": 242.1954231262207, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698769.317432, "available_gib": 242.13496017456055, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698771.3415475, "available_gib": 242.03434371948242, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789698773.3832676, "available_gib": 241.72362899780273, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698775.4252958, "available_gib": 241.35395431518555, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698777.4691627, "available_gib": 241.17080307006836, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698779.5117922, "available_gib": 240.81914138793945, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698781.5538497, "available_gib": 240.2495880126953, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698783.5959759, "available_gib": 240.0577278137207, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false} +{"time": 1789698785.637802, "available_gib": 239.45506286621094, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false} +{"time": 1789698787.6577628, "available_gib": 239.00636291503906, "oom_kill": 0, "gpu": "643, 85651, 0, 30", "stop": false} +{"time": 1789698789.6773047, "available_gib": 238.29972076416016, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698791.6970866, "available_gib": 233.71295166015625, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698793.7173653, "available_gib": 229.35329818725586, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698795.7371087, "available_gib": 225.2687644958496, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698797.7571712, "available_gib": 221.2604522705078, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698799.7774785, "available_gib": 217.34229278564453, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698801.7975748, "available_gib": 213.40470504760742, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698803.8178012, "available_gib": 209.46981048583984, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698805.838123, "available_gib": 205.5439682006836, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698807.8585641, "available_gib": 201.61297607421875, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698809.8787436, "available_gib": 197.6831398010254, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698811.8989053, "available_gib": 193.7314338684082, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698813.9194837, "available_gib": 189.7943000793457, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698815.939467, "available_gib": 185.83709335327148, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698817.95988, "available_gib": 181.88336563110352, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698819.979742, "available_gib": 177.91891479492188, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698821.9990664, "available_gib": 173.97349166870117, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789698824.019928, "available_gib": 173.15047073364258, "oom_kill": 0, "gpu": "76915, 85651, 0, 31", "stop": false} +{"time": 1789698826.0405757, "available_gib": 173.21429061889648, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698828.0614407, "available_gib": 173.24291610717773, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698830.0821567, "available_gib": 173.27059936523438, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698832.103856, "available_gib": 173.86737442016602, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698834.1246617, "available_gib": 173.88233184814453, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698836.1457129, "available_gib": 173.87153244018555, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789698838.1666973, "available_gib": 173.88751983642578, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698840.187669, "available_gib": 173.88742065429688, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698842.2085977, "available_gib": 173.9085350036621, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698844.2294323, "available_gib": 173.91327667236328, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698846.2497046, "available_gib": 173.90702438354492, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698848.270506, "available_gib": 173.91292572021484, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698850.2913115, "available_gib": 173.90917205810547, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698852.312283, "available_gib": 173.91419982910156, "oom_kill": 0, "gpu": "76915, 85651, 6, 32", "stop": false} +{"time": 1789698854.332851, "available_gib": 173.92702102661133, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698856.3530693, "available_gib": 173.92903900146484, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698858.3738403, "available_gib": 173.91302108764648, "oom_kill": 0, "gpu": "76915, 85651, 8, 32", "stop": false} +{"time": 1789698860.3946135, "available_gib": 173.92741775512695, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698862.4152138, "available_gib": 173.9324073791504, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698864.4356892, "available_gib": 173.94479751586914, "oom_kill": 0, "gpu": "76915, 85651, 4, 32", "stop": false} +{"time": 1789698866.4579444, "available_gib": 173.87903213500977, "oom_kill": 0, "gpu": "76915, 85651, 0, 32", "stop": false} +{"time": 1789698868.4783835, "available_gib": 173.8003158569336, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698870.4979591, "available_gib": 173.48406600952148, "oom_kill": 0, "gpu": "76427, 85651, 0, 32", "stop": false} +{"time": 1789698872.519003, "available_gib": 172.82804107666016, "oom_kill": 0, "gpu": "78247, 85651, 0, 32", "stop": false} +{"time": 1789698874.5388021, "available_gib": 172.81478118896484, "oom_kill": 0, "gpu": "78391, 85651, 0, 32", "stop": false} +{"time": 1789698876.5583742, "available_gib": 172.78876113891602, "oom_kill": 0, "gpu": "78391, 85651, 0, 32", "stop": false} +{"time": 1789698878.5790184, "available_gib": 172.7807273864746, "oom_kill": 0, "gpu": "79611, 85651, 100, 40", "stop": false} +{"time": 1789698880.5994496, "available_gib": 172.76989364624023, "oom_kill": 0, "gpu": "79611, 85651, 100, 42", "stop": false} +{"time": 1789698882.620308, "available_gib": 172.75430297851562, "oom_kill": 0, "gpu": "79611, 85651, 100, 43", "stop": false} +{"time": 1789698884.6409009, "available_gib": 172.74323272705078, "oom_kill": 0, "gpu": "79615, 85651, 0, 35", "stop": false} +{"time": 1789698886.6611326, "available_gib": 172.77804565429688, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698888.681134, "available_gib": 172.79762649536133, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698890.7010674, "available_gib": 172.75135040283203, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789698892.7217126, "available_gib": 172.77525329589844, "oom_kill": 0, "gpu": "76761, 85651, 0, 34", "stop": false} +{"time": 1789698894.7425878, "available_gib": 172.78721237182617, "oom_kill": 0, "gpu": "76841, 85651, 0, 33", "stop": false} +{"time": 1789698896.7630284, "available_gib": 172.94769287109375, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789698898.7830164, "available_gib": 172.9618263244629, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789698900.8028846, "available_gib": 173.00798797607422, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789698902.8227024, "available_gib": 173.0347442626953, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789698904.8435998, "available_gib": 173.07442092895508, "oom_kill": 0, "gpu": "80123, 85651, 12, 33", "stop": false} +{"time": 1789698906.8641193, "available_gib": 173.00585174560547, "oom_kill": 0, "gpu": "80129, 85651, 0, 33", "stop": false} +{"time": 1789698908.8844786, "available_gib": 172.9863395690918, "oom_kill": 0, "gpu": "80175, 85651, 0, 33", "stop": false} +{"time": 1789698910.9042525, "available_gib": 172.9756317138672, "oom_kill": 0, "gpu": "80175, 85651, 0, 33", "stop": false} +{"time": 1789698912.9238913, "available_gib": 172.99994277954102, "oom_kill": 0, "gpu": "80175, 85651, 0, 33", "stop": false} +{"time": 1789698914.9437, "available_gib": 173.02386474609375, "oom_kill": 0, "gpu": "80175, 85651, 0, 33", "stop": false} +{"time": 1789698916.9687424, "available_gib": 173.03638076782227, "oom_kill": 0, "gpu": "80179, 85651, 100, 38", "stop": false} +{"time": 1789698918.9929106, "available_gib": 173.03876876831055, "oom_kill": 0, "gpu": "80181, 85651, 92, 39", "stop": false} +{"time": 1789698921.0157156, "available_gib": 173.06135177612305, "oom_kill": 0, "gpu": "80181, 85651, 100, 40", "stop": false} +{"time": 1789698923.0401592, "available_gib": 172.91560745239258, "oom_kill": 0, "gpu": "80419, 85651, 100, 44", "stop": false} +{"time": 1789698925.0639243, "available_gib": 172.9213638305664, "oom_kill": 0, "gpu": "80419, 85651, 100, 45", "stop": false} +{"time": 1789698927.088088, "available_gib": 172.9150733947754, "oom_kill": 0, "gpu": "80419, 85651, 100, 46", "stop": false} +{"time": 1789698929.1123986, "available_gib": 172.91910934448242, "oom_kill": 0, "gpu": "80419, 85651, 100, 47", "stop": false} +{"time": 1789698931.1366441, "available_gib": 172.9725799560547, "oom_kill": 0, "gpu": "80419, 85651, 100, 47", "stop": false} +{"time": 1789698933.1609597, "available_gib": 172.9855613708496, "oom_kill": 0, "gpu": "80419, 85651, 100, 49", "stop": false} +{"time": 1789698935.1866493, "available_gib": 172.9901580810547, "oom_kill": 0, "gpu": "81065, 85651, 100, 50", "stop": false} +{"time": 1789698937.2117255, "available_gib": 173.0001983642578, "oom_kill": 0, "gpu": "82029, 85651, 100, 45", "stop": false} +{"time": 1789698939.2370842, "available_gib": 173.0097427368164, "oom_kill": 0, "gpu": "82029, 85651, 100, 45", "stop": false} +{"time": 1789698941.2622855, "available_gib": 173.01856231689453, "oom_kill": 0, "gpu": "82029, 85651, 100, 45", "stop": false} +{"time": 1789698943.2875965, "available_gib": 173.01775360107422, "oom_kill": 0, "gpu": "82029, 85651, 100, 45", "stop": false} +{"time": 1789698945.3126564, "available_gib": 173.01601028442383, "oom_kill": 0, "gpu": "82029, 85651, 100, 46", "stop": false} +{"time": 1789698947.337592, "available_gib": 173.01439666748047, "oom_kill": 0, "gpu": "82029, 85651, 100, 46", "stop": false} +{"time": 1789698949.3626537, "available_gib": 173.01418685913086, "oom_kill": 0, "gpu": "82029, 85651, 100, 46", "stop": false} +{"time": 1789698951.38797, "available_gib": 173.02657318115234, "oom_kill": 0, "gpu": "82029, 85651, 100, 46", "stop": false} +{"time": 1789698953.4083169, "available_gib": 173.0288429260254, "oom_kill": 0, "gpu": "82029, 85651, 0, 41", "stop": false} +{"time": 1789698955.4286387, "available_gib": 173.02482223510742, "oom_kill": 0, "gpu": "82029, 85651, 0, 41", "stop": false} +{"time": 1789698957.4486737, "available_gib": 173.02771377563477, "oom_kill": 0, "gpu": "82029, 85651, 0, 40", "stop": false} +{"time": 1789698959.468538, "available_gib": 173.02423477172852, "oom_kill": 0, "gpu": "82029, 85651, 0, 40", "stop": false} +{"time": 1789698961.4885864, "available_gib": 173.03007888793945, "oom_kill": 0, "gpu": "82029, 85651, 0, 39", "stop": false} +{"time": 1789698963.5087643, "available_gib": 173.02819442749023, "oom_kill": 0, "gpu": "82029, 85651, 0, 39", "stop": false} +{"time": 1789698965.5290365, "available_gib": 173.0245475769043, "oom_kill": 0, "gpu": "82029, 85651, 0, 38", "stop": false} +{"time": 1789698967.5490716, "available_gib": 173.02976989746094, "oom_kill": 0, "gpu": "82029, 85651, 0, 38", "stop": false} +{"time": 1789698969.5692663, "available_gib": 173.02922439575195, "oom_kill": 0, "gpu": "82029, 85651, 0, 38", "stop": false} +{"time": 1789698971.5892925, "available_gib": 173.02708435058594, "oom_kill": 0, "gpu": "82029, 85651, 0, 37", "stop": false} +{"time": 1789698973.6092043, "available_gib": 173.02328491210938, "oom_kill": 0, "gpu": "82029, 85651, 0, 37", "stop": false} +{"time": 1789698975.629257, "available_gib": 173.02639389038086, "oom_kill": 0, "gpu": "82029, 85651, 0, 37", "stop": false} +{"time": 1789698977.6489754, "available_gib": 173.02574157714844, "oom_kill": 0, "gpu": "82029, 85651, 0, 37", "stop": false} +{"time": 1789698979.6686509, "available_gib": 173.02384567260742, "oom_kill": 0, "gpu": "82029, 85651, 0, 36", "stop": false} +{"time": 1789698981.6883833, "available_gib": 173.03147888183594, "oom_kill": 0, "gpu": "82029, 85651, 0, 36", "stop": false} +{"time": 1789698983.7081096, "available_gib": 173.01952743530273, "oom_kill": 0, "gpu": "82029, 85651, 0, 36", "stop": false} +{"time": 1789698985.7279449, "available_gib": 173.0206756591797, "oom_kill": 0, "gpu": "82029, 85651, 0, 36", "stop": false} +{"time": 1789698987.7474644, "available_gib": 173.01938247680664, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698989.7669823, "available_gib": 173.02622604370117, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698991.7865682, "available_gib": 173.01847076416016, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698993.806216, "available_gib": 173.02779006958008, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698995.8259676, "available_gib": 173.01957321166992, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698997.8460538, "available_gib": 173.0227279663086, "oom_kill": 0, "gpu": "82029, 85651, 0, 35", "stop": false} +{"time": 1789698999.8657992, "available_gib": 173.01766204833984, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699001.8856435, "available_gib": 173.0223274230957, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699003.9054878, "available_gib": 173.0254364013672, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699005.9253855, "available_gib": 173.02705001831055, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699007.945158, "available_gib": 173.02701950073242, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699009.9652565, "available_gib": 173.02615356445312, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699011.98536, "available_gib": 173.02739334106445, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699014.0051653, "available_gib": 173.0303840637207, "oom_kill": 0, "gpu": "82029, 85651, 0, 34", "stop": false} +{"time": 1789699016.0250003, "available_gib": 173.02931594848633, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699018.0447857, "available_gib": 173.02694702148438, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699020.0646138, "available_gib": 173.0302276611328, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699022.084513, "available_gib": 173.02870178222656, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699024.1043646, "available_gib": 173.0284538269043, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699026.1244905, "available_gib": 173.03363800048828, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699028.1439598, "available_gib": 173.0325469970703, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699030.1636896, "available_gib": 173.03043365478516, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699032.183471, "available_gib": 173.0263214111328, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699034.2031057, "available_gib": 173.01843643188477, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699036.2231333, "available_gib": 173.02396774291992, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699038.2431438, "available_gib": 173.02379989624023, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699040.2632885, "available_gib": 173.03008651733398, "oom_kill": 0, "gpu": "82029, 85651, 0, 33", "stop": false} +{"time": 1789699042.2832162, "available_gib": 173.03044891357422, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699044.303224, "available_gib": 173.02885055541992, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699046.3234336, "available_gib": 173.03288650512695, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699048.3437572, "available_gib": 173.02514266967773, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699050.3640664, "available_gib": 173.03996658325195, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699052.3841255, "available_gib": 173.0434341430664, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699054.4042444, "available_gib": 173.0456771850586, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699056.4242733, "available_gib": 173.0513916015625, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699058.4442263, "available_gib": 173.05170822143555, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699060.4642491, "available_gib": 173.05607223510742, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699062.4840407, "available_gib": 173.04257202148438, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699064.5037813, "available_gib": 173.02481079101562, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699066.523792, "available_gib": 173.0170555114746, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699068.5436466, "available_gib": 173.0222396850586, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699070.5636365, "available_gib": 173.0317268371582, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699072.5836728, "available_gib": 173.03863143920898, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699074.6036873, "available_gib": 173.03907012939453, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699076.6236837, "available_gib": 173.03926849365234, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699078.6435843, "available_gib": 173.0428237915039, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699080.6636631, "available_gib": 173.04558944702148, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699082.6837516, "available_gib": 173.04774856567383, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699084.70407, "available_gib": 173.0488624572754, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699086.7241883, "available_gib": 173.05421829223633, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699088.7443526, "available_gib": 173.05388641357422, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699090.764633, "available_gib": 173.05226135253906, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699092.7847042, "available_gib": 173.0488395690918, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699094.8048282, "available_gib": 173.04421615600586, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699096.8249772, "available_gib": 173.05382537841797, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699098.8451283, "available_gib": 173.05199432373047, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699100.8652503, "available_gib": 173.04828262329102, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699102.8852892, "available_gib": 173.0489387512207, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699104.9053323, "available_gib": 173.05001831054688, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699106.925421, "available_gib": 173.04602813720703, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699108.945708, "available_gib": 173.04939651489258, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699110.965652, "available_gib": 173.0514373779297, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699112.9856977, "available_gib": 173.05028915405273, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699115.0057292, "available_gib": 173.0475845336914, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699117.0259068, "available_gib": 173.0482406616211, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699119.0462375, "available_gib": 173.05077743530273, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699121.0662673, "available_gib": 173.05032348632812, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699123.0862777, "available_gib": 173.04265213012695, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699125.1062524, "available_gib": 173.04559707641602, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699127.126354, "available_gib": 173.0432586669922, "oom_kill": 0, "gpu": "82029, 85651, 0, 32", "stop": false} +{"time": 1789699129.146269, "available_gib": 173.04304885864258, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699131.1664004, "available_gib": 173.04404830932617, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699133.1864913, "available_gib": 173.04531860351562, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699135.2065778, "available_gib": 173.04535675048828, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699137.2268605, "available_gib": 173.04468536376953, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699139.2469645, "available_gib": 173.0467071533203, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699141.2672274, "available_gib": 173.0432891845703, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699143.2872388, "available_gib": 173.05076599121094, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699145.307178, "available_gib": 173.04270553588867, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699147.3273482, "available_gib": 173.04224014282227, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699149.3473735, "available_gib": 173.0479736328125, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699151.3675382, "available_gib": 173.04681777954102, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699153.3875065, "available_gib": 173.0449981689453, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699155.4074852, "available_gib": 173.03551483154297, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699157.4275417, "available_gib": 173.03624725341797, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699159.44749, "available_gib": 173.03990936279297, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699161.4676747, "available_gib": 174.74375534057617, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699163.4883516, "available_gib": 174.74128341674805, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699165.5085652, "available_gib": 175.53340911865234, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699167.528961, "available_gib": 175.52461624145508, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699169.5493548, "available_gib": 175.52721405029297, "oom_kill": 0, "gpu": "82029, 85651, 0, 31", "stop": false} +{"time": 1789699176.422113, "available_gib": 190.14447021484375, "oom_kill": 0, "gpu": "3, 85651, 0, 31", "stop": false} +{"time": 1789699178.4453363, "available_gib": 242.4462432861328, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false} +{"time": 1789699180.4677014, "available_gib": 242.36112213134766, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789699182.4918027, "available_gib": 242.28740310668945, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789699184.5162215, "available_gib": 242.26057815551758, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false} +{"time": 1789699186.5411034, "available_gib": 242.2496109008789, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789699188.565741, "available_gib": 242.1990089416504, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789699190.5900302, "available_gib": 242.13172912597656, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789699192.6151948, "available_gib": 241.99724578857422, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false} +{"time": 1789699194.6575375, "available_gib": 241.70180130004883, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false} +{"time": 1789699196.7025993, "available_gib": 241.3562889099121, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false} +{"time": 1789699198.744729, "available_gib": 241.16839981079102, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false} +{"time": 1789699200.786438, "available_gib": 240.83783340454102, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false} +{"time": 1789699202.828659, "available_gib": 240.20919036865234, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false} +{"time": 1789699204.873085, "available_gib": 239.9847755432129, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false} +{"time": 1789699206.9188688, "available_gib": 239.36572647094727, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false} +{"time": 1789699208.9390886, "available_gib": 238.9336166381836, "oom_kill": 0, "gpu": "2873, 85651, 0, 29", "stop": false} +{"time": 1789699210.9585626, "available_gib": 237.42334747314453, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699212.9778147, "available_gib": 232.8353729248047, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699214.9978516, "available_gib": 228.88347244262695, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699217.0177827, "available_gib": 224.88120651245117, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699219.0375445, "available_gib": 220.8877182006836, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699221.057924, "available_gib": 216.8581314086914, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699223.0776703, "available_gib": 212.8553695678711, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699225.0975223, "available_gib": 208.81986618041992, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699227.117495, "available_gib": 204.67229080200195, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699229.1374993, "available_gib": 200.6156463623047, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699231.1575491, "available_gib": 196.59537506103516, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699233.1772435, "available_gib": 192.61323165893555, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699235.1972623, "available_gib": 188.58648681640625, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699237.2170012, "available_gib": 184.57256698608398, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699239.237018, "available_gib": 180.54946517944336, "oom_kill": 0, "gpu": "4245, 85651, 0, 30", "stop": false} +{"time": 1789699241.2569234, "available_gib": 176.51204299926758, "oom_kill": 0, "gpu": "4245, 85651, 0, 31", "stop": false} +{"time": 1789699243.2777386, "available_gib": 174.25994491577148, "oom_kill": 0, "gpu": "4363, 85651, 15, 31", "stop": false} +{"time": 1789699245.2984893, "available_gib": 174.2334327697754, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699247.3195407, "available_gib": 174.25234603881836, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699249.340046, "available_gib": 174.26653671264648, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699251.3610094, "available_gib": 174.26525497436523, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699253.3823578, "available_gib": 174.28969955444336, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699255.4044833, "available_gib": 174.29557037353516, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699257.4260921, "available_gib": 174.32069778442383, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699259.4473388, "available_gib": 174.3233528137207, "oom_kill": 0, "gpu": "76915, 85651, 5, 31", "stop": false} +{"time": 1789699261.4683244, "available_gib": 174.32538223266602, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699263.4894924, "available_gib": 174.38761520385742, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699265.510466, "available_gib": 174.37882232666016, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789699267.5313315, "available_gib": 174.37767791748047, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699269.5522597, "available_gib": 174.3699073791504, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699271.5731544, "available_gib": 174.3562889099121, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699273.5937147, "available_gib": 174.40947341918945, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699275.6145303, "available_gib": 174.39099502563477, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699277.6357915, "available_gib": 174.40030670166016, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699279.6567283, "available_gib": 174.40108489990234, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699281.678276, "available_gib": 174.41379165649414, "oom_kill": 0, "gpu": "76915, 85651, 4, 31", "stop": false} +{"time": 1789699283.6991897, "available_gib": 174.4610939025879, "oom_kill": 0, "gpu": "76915, 85651, 6, 31", "stop": false} +{"time": 1789699285.7248394, "available_gib": 174.46648788452148, "oom_kill": 0, "gpu": "76915, 85651, 0, 31", "stop": false} +{"time": 1789699287.7454658, "available_gib": 174.38458633422852, "oom_kill": 0, "gpu": "76427, 85651, 0, 31", "stop": false} +{"time": 1789699289.7651424, "available_gib": 174.01377487182617, "oom_kill": 0, "gpu": "76427, 85651, 0, 31", "stop": false} +{"time": 1789699291.7871606, "available_gib": 173.7390365600586, "oom_kill": 0, "gpu": "76427, 85651, 0, 31", "stop": false} +{"time": 1789699293.8075342, "available_gib": 173.08771896362305, "oom_kill": 0, "gpu": "78391, 85651, 0, 31", "stop": false} +{"time": 1789699295.8274386, "available_gib": 173.07221221923828, "oom_kill": 0, "gpu": "78391, 85651, 0, 31", "stop": false} +{"time": 1789699297.847325, "available_gib": 173.05772018432617, "oom_kill": 0, "gpu": "78391, 85651, 0, 31", "stop": false} +{"time": 1789699299.8682976, "available_gib": 173.04212951660156, "oom_kill": 0, "gpu": "79611, 85651, 100, 42", "stop": false} +{"time": 1789699301.8895893, "available_gib": 173.04708862304688, "oom_kill": 0, "gpu": "79611, 85651, 100, 43", "stop": false} +{"time": 1789699303.910048, "available_gib": 173.06679153442383, "oom_kill": 0, "gpu": "79613, 85651, 68, 39", "stop": false} +{"time": 1789699305.9303918, "available_gib": 173.07761764526367, "oom_kill": 0, "gpu": "79615, 85651, 0, 34", "stop": false} +{"time": 1789699307.9507058, "available_gib": 173.12001037597656, "oom_kill": 0, "gpu": "79615, 85651, 0, 33", "stop": false} +{"time": 1789699309.9707806, "available_gib": 173.1412010192871, "oom_kill": 0, "gpu": "79615, 85651, 0, 33", "stop": false} +{"time": 1789699311.9934156, "available_gib": 173.16471481323242, "oom_kill": 0, "gpu": "76979, 85651, 51, 40", "stop": false} +{"time": 1789699314.0141187, "available_gib": 173.26668548583984, "oom_kill": 0, "gpu": "76795, 85651, 0, 33", "stop": false} +{"time": 1789699316.0347028, "available_gib": 173.47204971313477, "oom_kill": 0, "gpu": "76849, 85651, 0, 33", "stop": false} +{"time": 1789699318.0550356, "available_gib": 173.4830322265625, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789699320.074777, "available_gib": 173.44154739379883, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789699322.0946026, "available_gib": 173.43824005126953, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789699324.1150944, "available_gib": 173.36431121826172, "oom_kill": 0, "gpu": "80119, 85651, 0, 33", "stop": false} +{"time": 1789699326.1359715, "available_gib": 173.24655532836914, "oom_kill": 0, "gpu": "80129, 85651, 1, 33", "stop": false} +{"time": 1789699328.1563468, "available_gib": 173.23368072509766, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699330.176313, "available_gib": 173.17479705810547, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699332.196048, "available_gib": 173.1882667541504, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699334.2162025, "available_gib": 173.19011306762695, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699336.2361436, "available_gib": 173.16001892089844, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699338.256036, "available_gib": 173.18156051635742, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699340.2761614, "available_gib": 173.19681930541992, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699342.2961998, "available_gib": 173.218994140625, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699344.316221, "available_gib": 173.2559814453125, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699346.3366456, "available_gib": 173.2540626525879, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699348.3572412, "available_gib": 173.26934432983398, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699350.3776436, "available_gib": 173.2864227294922, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699352.398027, "available_gib": 173.28744888305664, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699354.4182692, "available_gib": 173.27970504760742, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699356.4386253, "available_gib": 173.2762222290039, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699358.4588006, "available_gib": 173.27862930297852, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699360.4790263, "available_gib": 173.27960205078125, "oom_kill": 0, "gpu": "80175, 85651, 0, 32", "stop": false} +{"time": 1789699362.5037036, "available_gib": 173.25759887695312, "oom_kill": 0, "gpu": "80181, 85651, 100, 38", "stop": false} +{"time": 1789699364.5242743, "available_gib": 173.26276016235352, "oom_kill": 0, "gpu": "80181, 85651, 0, 33", "stop": false} +{"time": 1789699366.5446012, "available_gib": 173.24929428100586, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699368.5647185, "available_gib": 173.25961303710938, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699370.5847807, "available_gib": 173.26060104370117, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699372.6047556, "available_gib": 173.26032257080078, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699374.624905, "available_gib": 173.2593231201172, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699376.6449807, "available_gib": 173.26322555541992, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699378.6651788, "available_gib": 173.2634048461914, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699380.6851304, "available_gib": 173.26718521118164, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699382.7051644, "available_gib": 173.2645721435547, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} +{"time": 1789699384.7251182, "available_gib": 173.26424407958984, "oom_kill": 0, "gpu": "80181, 85651, 0, 32", "stop": false} diff --git a/audit/runtime/source-hashes.json b/audit/runtime/source-hashes.json new file mode 100644 index 0000000..ea9e52b --- /dev/null +++ b/audit/runtime/source-hashes.json @@ -0,0 +1 @@ +{"models/qwen4_exp/nvidia/ngram_embedding.py": "01c2cb7c691c9a8a448e57619fdb97bfe1c3230e7be3c92dd4642d3f7173a499", "models/qwen4_exp/nvidia/qsa.py": "f7307663d341e3fdf5ee2ae866aaea449f1e89564938bffd66fb3211147b4f3f", "models/qwen4_exp/nvidia/mtp.py": "35dc7e4b4cf124ce79a7bfc65a5f198c9edc622f1692bf6b24c91486112b7d99", "model_executor/layers/quantization/modelopt.py": "e65b339d573e1650508a9ab109c29f098e0a1cc799fbe04737e96596604fff02", "config/engram.py": "34043a22385a041d4e9be8857f64174c990c2ca8308b97b6a952b5ab96c20ee5"} diff --git a/audit/runtime/weights-verification.jsonl b/audit/runtime/weights-verification.jsonl new file mode 100644 index 0000000..6508c0c --- /dev/null +++ b/audit/runtime/weights-verification.jsonl @@ -0,0 +1,11 @@ +{"file": "model-00001-of-00010.safetensors", "bytes": 3115991696, "sha256": "63fde954be6f08b49b876f4f70a0ad0bcfee71aff7b1faa33779b6b32feca2a2", "passed": true} +{"file": "model-00003-of-00010.safetensors", "bytes": 10005364728, "sha256": "3218ddc129258e91a721a8e81329ca8f00588332ec721bd8d2cf7d20e324bf47", "passed": true} +{"file": "model-00004-of-00010.safetensors", "bytes": 10006060112, "sha256": "55e2bdf6a3a1f6e65270787f65c63b2a2a56b308b54eb1fb318dfabc9b48b141", "passed": true} +{"file": "model-00007-of-00010.safetensors", "bytes": 10005956544, "sha256": "8cf3fa05c04cb2e060963b69bb01f7a7b9f43c58435fbe380836a0457a19cd3e", "passed": true} +{"file": "model-00010-of-00010.safetensors", "bytes": 128587536, "sha256": "0d49b0cf7c15bf9b5bd1e17403b36727315860a8f2a2e0f5a53655547084f640", "passed": true} +{"file": "model-00006-of-00010.safetensors", "bytes": 10005375328, "sha256": "38f65a9d11090428e139cc60d0587e8e5433ea2ce79820883c583d7cfcc53130", "passed": true} +{"file": "model-fp8-mtp-ple.safetensors", "bytes": 53717551730, "sha256": "3525520c8602d850003eb1960aec0b64291dae33b83f8b65dd639a451df78823", "passed": true} +{"file": "model-00005-of-00010.safetensors", "bytes": 10005362392, "sha256": "d3c169d3694bfba846455d01e48f047d770f78d071c5a4d22f129389411d686e", "passed": true} +{"file": "model-00009-of-00010.safetensors", "bytes": 5679113808, "sha256": "eaff6a87ece6fbfd6ad0fbaa0a5cf9ff542f8076acba38e62410b1ed55e12cc3", "passed": true} +{"file": "model-00002-of-00010.safetensors", "bytes": 10005510304, "sha256": "4dafaef62a908e49e0d92c7c2a3fa99f4d9651fdeeb0ca09f09a9b40095af4e0", "passed": true} +{"file": "model-00008-of-00010.safetensors", "bytes": 10005375200, "sha256": "53d1f80746aa0cc0a7c2f4836587002a8a4dd72c827432cd2e43abae3c3899e4", "passed": true} diff --git a/audit/runtime/weights-verified b/audit/runtime/weights-verified new file mode 100644 index 0000000..b0aad4d --- /dev/null +++ b/audit/runtime/weights-verified @@ -0,0 +1 @@ +passed diff --git a/audit/source-review.md b/audit/source-review.md new file mode 100644 index 0000000..17c2796 --- /dev/null +++ b/audit/source-review.md @@ -0,0 +1,32 @@ +# RTX 6000D 部署审计 + +状态：所选路径源码审计及运行验收完成。2026-09-18。 + +## 源码基线 + +- 参考项目： https://github.com/tpurtell/sm12x-exl3-qwen3.8-flash-next +- 审计 revision：`847f45b1730069eab4bdb5e29361f9ea94ac5e05`。 +- 实际采用的官方 vLLM：`0bfc7a15d095fe83ecc82b50561a93c177fece2d`。 +- 官方镜像 digest：`sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28`；部署前核对 AMD64 镜像源码。 +- NVIDIA 模型 revision：`fc694b54fb0174e0913e6adf86691ef85a4ead47`，从已有 Spark 缓存通过内网复制，避免外网代理限流。 + +## 审计决定 + +参考项目固定了依赖版本，并为 resident PLE、工具约束和量化路径提供测试，这是有价值的参考。但其 Dockerfile 同时引入 EXL3 来源镜像、B12x 自定义注意力内核、mmap 回补及其他分支，不能直接作为本机最小部署。 + +1. 不使用原项目 Dockerfile、启动脚本及预构建社区镜像。自行编写仅服务 NVIDIA NVFP4 的 Compose。 +2. 原项目的 ModelOpt FP8 PLE 修复、MTP 层名映射和 block-FP8 分派已经存在于固定新版官方 vLLM，因此不重复回补。 +3. 新版 `ngram_embedding.py` 的 `Qwen4ExpPLEPinnedHostEmbedding` 直接在锁页 CPU 内存分配 PLE，通过 UVA Triton 查表，side stream 预取并在使用前等待；保留原有 FP8 数据及全局缩放，不额外量化。 +4. 使用显式 `--engram-config '{"cpu_offload":true}'`，不照搬旧版环境变量和子进程加载修复。新版已不采用该旧 resident 子进程结构。 +5. 不移植 EXL3、mmap、B12x 内核、全局 token embedding monkeypatch。主机内存充足，无需为节约 PLE 内存增加这些路径。 +6. 固定新版已有原生 FP8 QSA 支持；先以 BF16 KV 建立基线，FP8 KV 仅在数值和端到端验收后启用。 +7. 对此前已定位的 Mamba prefix block alignment 问题保留有来源的窄补丁，先核对精确文件哈希；不采用 GB10 专用 shared-memory 限制补丁。 +8. 原项目宿主网络、无认证默认值不适合作为本机对外服务默认配置。使用显式端口映射、独立 API key、只读模型挂载、内存上限。 + +## 硬件与限制 + +实测：VMware 虚拟机、16 vCPU、247 GiB RAM、85651 MiB GPU 显存。原项目 96GB 显卡的显存预算和速度数据不作为本机验收证据。虚拟 PCIe 配置显示的链路不用于推断实际带宽。 + +已验证 CUDA 运算、PLE 锁页查表及量化缩放；依次完成 32K BF16 eager、128K FP8 eager、128K FP8 FULL CUDA Graph 验收。最终启用 128K 和 CUDA Graph；MTP 未启用。文字、工具调用、图片、视频、127988 token 输入检索均通过。详见 [验收结果](results.md)。 + +本审计是所选执行路径的代码审查和测试计划，不是全部第三方依赖的安全认证。 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..f5c7502 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,45 @@ +name: qwen38-flash-rtx6000d +services: + vllm: + image: local/qwen38-flash-6000d:0bfc7a15 + build: . + container_name: qwen38-flash-6000d + gpus: all + shm_size: 8g + mem_limit: 210g + memswap_limit: 210g + restart: unless-stopped + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${API_PORT:-8000}:8000" + environment: + HF_HUB_OFFLINE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_WORKER_MULTIPROC_METHOD: spawn + OMP_NUM_THREADS: "8" + TORCHINDUCTOR_COMPILE_THREADS: "2" + CUTE_DSL_ARCH: sm_120a + volumes: + - /data/flash-next/models/nvidia-Qwen3.8-Flash-Next-NVFP4:/model:ro + - /data/flash-next/cache:/root/.cache + - ./secrets/api-key:/run/secrets/api-key:ro + entrypoint: ["/bin/bash", "-lc"] + command: + - >- + exec vllm serve /model + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 --dtype bfloat16 + --engram-config '{"cpu_offload":true}' + --kv-cache-dtype fp8 --gpu-memory-utilization 0.96 + --max-model-len 131072 --max-num-seqs 1 --max-num-batched-tokens 2048 + --enable-chunked-prefill --enable-prefix-caching + --compilation-config '{"mode":0,"cudagraph_mode":"FULL","cudagraph_capture_sizes":[1]}' --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/api-key)" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30m diff --git a/docs/lessons.md b/docs/lessons.md new file mode 100644 index 0000000..8f5cfe4 --- /dev/null +++ b/docs/lessons.md @@ -0,0 +1,38 @@ +# 部署心得与后续优化 + +记录日期：2026-09-18。结论对应本仓库固定的镜像、模型和实测，不自动适用于后续 nightly。 + +## 先核对硬件，再计算预算 + +这台 VMware 机器提供 247 GiB 可用总内存和 85651 MiB GPU 显存，不能直接套用社区 96GB 显卡的预算。虚拟 PCIe 链路显示也不能代表物理吞吐，本次通过锁页内存复制测得约 57.81 GB/s。 + +CPU 内存充足适合存放约 47.7 GiB 的 PLE 表。官方实现保留 FP8 数据及缩放，分配锁页主机内存，通过 UVA 访问。普通独立显卡主机与 Spark 统一内存架构不同，配置应分别维护。 + +## 优先减少补丁 + +社区方案有参考价值，但旧版本所需的 resident PLE、FP8 和 MTP 修复可能已经进入新版本。先逐项对照固定官方源码，再决定是否回补。本项目没有引入社区 EXL3 镜像、B12x 自定义内核或 mmap 分支，仅保留两处带源码哈希保护的 Mamba prefix alignment 修复。 + +## CUDA Graph 要实际对照 + +本机 eager 与原生 FULL Graph 在相同输入、固定 256 输出 token 下，三次测量中位数分别约 22.0 和 77.1 token/s，正文和思考文本一致。这个收益不能直接套到 Spark，也不代表长输入预填充快了 3.5 倍：127988 token 首次输入两组首 token 都约 14.14 秒。 + +因此区分首 token 延迟、解码速度、前缀缓存命中和请求总耗时。固定长度计时包含思考 token，输出截断不应被当成质量测试。 + +## 基线先保住用户需要的能力 + +保留视觉编码器，已验证图片和短视频。128K、FP8 KV、单活动请求、不开 MTP 是本次选定基线。仍有空闲主机内存不等于应该继续扩大 GPU KV 或加载更多模型；主机内存余量不能直接替代显存预算。 + +接下来可分别测试 MTP、并发、较长视频和真实业务质量，每次只调整一个主要变量，记录 OOM、首 token、速度、正确性和资源峰值。当前没有证据宣称更高并发、262K 或任意长视频稳定。 + +## 遇到的问题 + +- 驱动安装后缺少 `nvidia-smi`：本机补装 `nvidia-utils-595-server` 并加载当前内核模块后恢复。重装或升级前先核对正在运行的内核、模块和用户态工具版本。 +- Hugging Face 直连失败、代理限流：从 Spark 已有缓存内网复制，并以原始 blob SHA256 校验全部 11 个权重文件。 +- `--calculate-kv-scales` 不被固定版本识别：删除不支持的参数，以当前实际 CLI 和源码为准。 +- 启动期健康接口未就绪会连接重置：先等待健康检查成功，再开始验收。最终复测结果见运行回执。 + +## 维护方式 + +`main` 保存已验收基线。实验从独立分支开始，记录镜像 digest、模型 revision、配置、测试条件和原始回执，验证后再合并。升级前保留旧镜像与配置；故障时先恢复已验收版本。 + +历史 `audit/runtime/compose-*.yaml` 是实验快照，不是可直接覆盖生产的完整运维配置，其端口绑定和重启策略可能不同。最终部署以根目录 `compose.yaml` 为准。回退 eager 时仅将 Graph compilation 参数替换成 `--enforce-eager`，保留端口、鉴权和重启策略，并复测。 diff --git a/patches/Apache-2.0.txt b/patches/Apache-2.0.txt new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/patches/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/patches/fix_prefix_alignment.py b/patches/fix_prefix_alignment.py new file mode 100644 index 0000000..8b0ec1e --- /dev/null +++ b/patches/fix_prefix_alignment.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# Based on blazux/qwen3.8-Flash-DGX's Mamba alignment diagnosis, +# Copyright 2026 blazux. Independently guarded for pinned vLLM 0bfc7a15. +import ast +import hashlib +import importlib.util +import json +from pathlib import Path + +root = Path(importlib.util.find_spec('vllm').origin).parent +edits = json.loads(Path(__file__).with_name('prefix-edits.json').read_text()) +pending = [] +for edit in edits: + path = root / edit['file'] + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != edit['sha256']: + raise RuntimeError(f'Unsupported source: {path}') + source = data.decode() + if source.count(edit['before']) != 1: + raise RuntimeError(f'Ambiguous patch anchor: {path}') + changed = source.replace(edit['before'], edit['after']) + ast.parse(changed) + pending.append((path, changed)) +for path, changed in pending: + path.write_text(changed) +print('Applied two source-verified Mamba prefix alignment corrections') diff --git a/patches/prefix-edits.json b/patches/prefix-edits.json new file mode 100644 index 0000000..d2b9fe6 --- /dev/null +++ b/patches/prefix-edits.json @@ -0,0 +1,14 @@ +[ + { + "file": "v1/worker/gpu/model_states/mamba_hybrid.py", + "sha256": "573e130ac5587d4822fabf97adad1ad0fda02ff83c5e70df879a5bb175ae845c", + "before": "(new_req_data.num_computed_tokens - 1) // self.cache_config.block_size", + "after": "(new_req_data.num_computed_tokens - 1)\n // (self.cache_config.mamba_block_size or self.cache_config.block_size)" + }, + { + "file": "v1/core/sched/scheduler.py", + "sha256": "ee5800550e8d52ddc81c9770bb82cf0c9a47a22a96d44f6e6f7d2e67d9900795", + "before": " block_size = self.cache_config.block_size\n # The last block-aligned", + "after": " block_size = self.block_size\n # The last block-aligned" + } +] diff --git a/scripts/acceptance.py b/scripts/acceptance.py new file mode 100644 index 0000000..fa0f051 --- /dev/null +++ b/scripts/acceptance.py @@ -0,0 +1,65 @@ +"""Small live API acceptance suite; writes evidence without API credentials.""" +import argparse +import json +import time +import urllib.request +from pathlib import Path + +p = argparse.ArgumentParser() +p.add_argument('--url', default='http://127.0.0.1:8000') +p.add_argument('--key-file', type=Path, default=Path(__file__).resolve().parents[1] / 'secrets/api-key') +p.add_argument('--output', type=Path, required=True) +args = p.parse_args() +key = args.key_file.read_text().strip() + +def request(path, body=None): + payload = None if body is None else json.dumps(body, ensure_ascii=False).encode() + req = urllib.request.Request(args.url + path, data=payload, + headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'}) + return urllib.request.urlopen(req, timeout=600) + +args.output.parent.mkdir(parents=True, exist_ok=True) +with args.output.open('x') as out: + def record(data): + line = json.dumps(data, ensure_ascii=False) + out.write(line + '\n'); out.flush(); print(line, flush=True) + + record({'models': json.load(request('/v1/models'))}) + for name, text, expected in [ + ('math', 'Compute 17 * 19. Reply with only the integer.', '323'), + ('chinese', '请只回复：模型已就绪', '模型已就绪'), + ]: + body = dict(model='qwen3.8-flash-next', messages=[dict(role='user', content=text)], + temperature=0, reasoning_effort='low', max_tokens=1024, + stream=True, stream_options={'include_usage': True}) + start = time.monotonic(); first = None; content = ''; reasoning = ''; usage = {}; finish = None + with request('/v1/chat/completions', body) as response: + for raw in response: + if not raw.startswith(b'data: '): continue + data = raw[6:].strip() + if data == b'[DONE]': break + item = json.loads(data) + if item.get('usage'): usage = item['usage'] + for choice in item.get('choices', []): + delta = choice.get('delta', {}) + c = delta.get('content') or '' + r = delta.get('reasoning') or delta.get('reasoning_content') or '' + if (c or r) and first is None: first = time.monotonic() + content += c; reasoning += r + finish = choice.get('finish_reason') or finish + elapsed = time.monotonic() - start + passed = content.strip() == expected and finish == 'stop' + record(dict(test=name, passed=passed, content=content, reasoning_chars=len(reasoning), + ttft_s=None if first is None else first-start, elapsed_s=elapsed, + finish_reason=finish, usage=usage)) + if not passed: raise SystemExit('Acceptance failure: ' + name) + body = dict(model='qwen3.8-flash-next', temperature=0, reasoning_effort='low', max_tokens=1024, + messages=[dict(role='user',content='Use get_weather to check the weather in Shanghai.')], + tools=[dict(type='function',function=dict(name='get_weather',description='Get current weather', + parameters=dict(type='object',properties={'city':{'type':'string'}},required=['city'])))], + tool_choice='auto') + result = json.load(request('/v1/chat/completions',body)) + calls = result['choices'][0]['message'].get('tool_calls') or [] + passed = bool(calls) and calls[0]['function']['name']=='get_weather' + record(dict(test='tool',passed=passed,response=result)) + if not passed: raise SystemExit('Tool acceptance failure') diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..498bc8e --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,33 @@ +"""Fixed 256-token single-stream timing; not a task-quality score.""" +import argparse +import json +import time +import urllib.request +from pathlib import Path + +p=argparse.ArgumentParser(); p.add_argument('--output',type=Path,required=True); a=p.parse_args() +key=(Path(__file__).resolve().parents[1]/'secrets/api-key').read_text().strip() +body=dict(model='qwen3.8-flash-next',messages=[dict(role='user',content='Write a Python function that merges overlapping intervals. Explain the algorithm, edge cases and complexity.')], + temperature=0,seed=6000,reasoning_effort='low',max_tokens=256,ignore_eos=True, + stream=True,stream_options={'include_usage':True}) +with a.output.open('x') as out: + for i in range(4): + req=urllib.request.Request('http://127.0.0.1:8000/v1/chat/completions',data=json.dumps(body).encode(), + headers={'Content-Type':'application/json','Authorization':'Bearer '+key}) + start=time.monotonic(); first=None; usage={}; content=''; reasoning='' + with urllib.request.urlopen(req,timeout=600) as resp: + for raw in resp: + if not raw.startswith(b'data: '): continue + data=raw[6:].strip() + if data==b'[DONE]': break + x=json.loads(data); usage=x.get('usage') or usage + for c in x.get('choices',[]): + d=c.get('delta',{}); s=d.get('content') or ''; r=d.get('reasoning') or d.get('reasoning_content') or '' + if first is None and (s or r): first=time.monotonic() + content+=s; reasoning+=r + end=time.monotonic(); tokens=usage['completion_tokens'] + if tokens != 256 or first is None: raise RuntimeError('Incomplete timing sample') + row=dict(run=i,warmup=i==0,ttft_s=first-start,elapsed_s=end-start, + decode_tps=(tokens-1)/(end-first),usage=usage,content=content,reasoning=reasoning) + line=json.dumps(row); out.write(line+'\n'); out.flush() + print(json.dumps({k:v for k,v in row.items() if k not in ['content','reasoning']}),flush=True) diff --git a/scripts/long_context.py b/scripts/long_context.py new file mode 100644 index 0000000..9954e37 --- /dev/null +++ b/scripts/long_context.py @@ -0,0 +1,57 @@ +"""Token-counted retrieval and repeat-prefix acceptance, not a broad quality eval.""" +import argparse +import json +import random +import time +import urllib.request +from pathlib import Path + +p=argparse.ArgumentParser() +p.add_argument('--tokens',type=int,default=28000) +p.add_argument('--output',type=Path,required=True) +p.add_argument('--url',default='http://127.0.0.1:8000') +a=p.parse_args() +key=(Path(__file__).resolve().parents[1]/'secrets/api-key').read_text().strip() +def request(path,body): + return urllib.request.urlopen(urllib.request.Request(a.url+path, + data=json.dumps(body).encode(),headers={'Content-Type':'application/json', + 'Authorization':'Bearer '+key}),timeout=900) + +rng=random.Random(6000) +lines=['Internal inventory. Find the exact special verification code when asked.\n', + 'The special verification code is VIOLET-74219.\n'] +lines += [f'Record {i:06d}: batch {rng.randrange(10**8):08d}, location shelf {rng.randrange(1000):03d}, status archived.\n' for i in range(a.tokens//8+1000)] +question='\nWhat is the special verification code? Reply with only that code.' +def messages(n): + return [{'role':'user','content':''.join(lines[:n])+question}] +lo,hi=2,len(lines) +while lo < hi: + mid=(lo+hi+1)//2 + with request('/tokenize',dict(model='qwen3.8-flash-next',messages=messages(mid), + chat_template_kwargs={'reasoning_effort':'low'})) as resp: + count=json.load(resp)['count'] + if count<=a.tokens: lo=mid + else: hi=mid-1 +a.output.parent.mkdir(parents=True,exist_ok=True) +with a.output.open('x') as out: + for repetition in range(2): + start=time.monotonic(); first=None; content=''; usage={}; finish=None + with request('/v1/chat/completions',dict(model='qwen3.8-flash-next',messages=messages(lo), + temperature=0,reasoning_effort='low',max_tokens=2048,stream=True, + stream_options={'include_usage':True})) as resp: + for raw in resp: + if not raw.startswith(b'data: '): continue + data=raw[6:].strip() + if data==b'[DONE]': break + item=json.loads(data) + usage=item.get('usage') or usage + for c in item.get('choices',[]): + d=c.get('delta',{}) + if first is None and any(d.get(x) for x in ['content','reasoning','reasoning_content']): first=time.monotonic() + content+=d.get('content') or '' + finish=c.get('finish_reason') or finish + row=dict(repetition=repetition,content=content,usage=usage,finish_reason=finish, + ttft_s=None if first is None else first-start,elapsed_s=time.monotonic()-start, + passed=content.strip()=='VIOLET-74219' and finish=='stop') + line=json.dumps(row); out.write(line+'\n'); out.flush(); print(line,flush=True) + if not row['passed']: raise SystemExit('Long-context retrieval failed') diff --git a/scripts/monitor.py b/scripts/monitor.py new file mode 100644 index 0000000..a4b3642 --- /dev/null +++ b/scripts/monitor.py @@ -0,0 +1,22 @@ +"""Log host/GPU headroom; stop the test server on host OOM or low RAM.""" +import json +import subprocess +import time +from pathlib import Path + +def kv(path): + return {s.split()[0].rstrip(':'): int(s.split()[1]) for s in Path(path).read_text().splitlines() if len(s.split())>=2} + +baseline = kv('/proc/vmstat')['oom_kill'] +while True: + mem = kv('/proc/meminfo'); vm = kv('/proc/vmstat') + gpu = subprocess.run(['nvidia-smi','--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu', + '--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=10) + low = mem['MemAvailable'] < 8*1024*1024 + oom = vm['oom_kill'] > baseline + print(json.dumps(dict(time=time.time(),available_gib=mem['MemAvailable']/1024**2, + oom_kill=vm['oom_kill'],gpu=gpu.stdout.strip(),stop=low or oom)),flush=True) + if low or oom: + subprocess.run(['docker','stop','-t','10','qwen38-flash-6000d'],timeout=30) + raise SystemExit('Host memory safeguard stopped server') + time.sleep(2) diff --git a/scripts/multimodal.py b/scripts/multimodal.py new file mode 100644 index 0000000..ce8dc95 --- /dev/null +++ b/scripts/multimodal.py @@ -0,0 +1,31 @@ +"""Verify actual image/video ingestion using deterministic local probes.""" +import argparse +import base64 +import json +import time +import urllib.request +from pathlib import Path + +p=argparse.ArgumentParser() +p.add_argument('--output',type=Path,required=True) +p.add_argument('--assets',type=Path,default=Path('/data/flash-next/cache')) +a=p.parse_args() +key=(Path(__file__).resolve().parents[1]/'secrets/api-key').read_text().strip() +cases=[('image','image/png','probe.png','What is the main color of this image? Reply with one English word.', ['red']), + ('video','video/mp4','probe.mp4','List the background colors in chronological order. Reply only with the three English color names.', ['red','green','blue'])] +with a.output.open('x') as out: + for kind,mime,name,prompt,colors in cases: + url='data:'+mime+';base64,'+base64.b64encode((a.assets/name).read_bytes()).decode() + part=kind+'_url' + body=dict(model='qwen3.8-flash-next',temperature=0,reasoning_effort='low',max_tokens=1024, + messages=[dict(role='user',content=[dict(type='text',text=prompt),{'type':part,part:{'url':url}}])]) + start=time.monotonic() + req=urllib.request.Request('http://127.0.0.1:8000/v1/chat/completions',data=json.dumps(body).encode(), + headers={'Content-Type':'application/json','Authorization':'Bearer '+key}) + with urllib.request.urlopen(req,timeout=600) as response: result=json.load(response) + choice=result['choices'][0]; content=choice['message'].get('content') or '' + pos=[content.lower().find(c) for c in colors] + passed=all(x>=0 for x in pos) and pos==sorted(pos) and choice['finish_reason']=='stop' + row=dict(test=kind,passed=passed,elapsed_s=time.monotonic()-start,response=result) + line=json.dumps(row,ensure_ascii=False); out.write(line+'\n'); out.flush(); print(line,flush=True) + if not passed: raise SystemExit('Multimodal probe failed: '+kind)