From 1e2f48d1a4930a5563389426ed206c2561825296 Mon Sep 17 00:00:00 2001 From: ruihan Date: Thu, 17 Sep 2026 22:09:58 +0800 Subject: [PATCH] Enable validated prefix caching and record DGX Spark optimization benchmarks --- .dockerignore | 3 + Dockerfile | 4 + README.md | 22 +++- THIRD_PARTY_NOTICES.md | 3 + compose.yaml | 4 +- configs/no-prefix.yaml | 17 +++ docs/architecture.md | 3 +- docs/benchmark-method.md | 45 ++++++++ docs/optimization-results.md | 64 ++++++++++++ docs/packaging-validation.md | 12 +++ docs/provenance.md | 9 +- docs/results/eager-long-1024.jsonl | 8 ++ docs/results/eager.jsonl | 21 ++++ docs/results/graph2.jsonl | 23 ++++ docs/results/prefix-eager-check.jsonl | 6 ++ docs/results/prefix-eager.jsonl | 21 ++++ experiments/graph/Dockerfile | 2 + experiments/graph/README.md | 23 ++++ experiments/graph/compose.yaml | 21 ++++ experiments/graph/spark_ngram_adapter.py | 94 +++++++++++++++++ experiments/graph/test_disk_adapter.py | 79 ++++++++++++++ patches/fix_mamba_prefix_blocks.py | 28 +++++ scripts/benchmark.py | 128 +++++++++++++++++++++++ scripts/benchmark.sh | 3 + scripts/prefix-check.py | 39 +++++++ scripts/start.sh | 4 +- scripts/summarize-benchmark.py | 26 +++++ scripts/test-prefix.sh | 3 + 28 files changed, 707 insertions(+), 8 deletions(-) create mode 100644 configs/no-prefix.yaml create mode 100644 docs/benchmark-method.md create mode 100644 docs/optimization-results.md create mode 100644 docs/results/eager-long-1024.jsonl create mode 100644 docs/results/eager.jsonl create mode 100644 docs/results/graph2.jsonl create mode 100644 docs/results/prefix-eager-check.jsonl create mode 100644 docs/results/prefix-eager.jsonl create mode 100644 experiments/graph/Dockerfile create mode 100644 experiments/graph/README.md create mode 100644 experiments/graph/compose.yaml create mode 100644 experiments/graph/spark_ngram_adapter.py create mode 100644 experiments/graph/test_disk_adapter.py create mode 100644 patches/fix_mamba_prefix_blocks.py create mode 100644 scripts/benchmark.py create mode 100755 scripts/benchmark.sh create mode 100644 scripts/prefix-check.py create mode 100644 scripts/summarize-benchmark.py create mode 100755 scripts/test-prefix.sh diff --git a/.dockerignore b/.dockerignore index bf9e0ee..35e253e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,6 @@ !patches/spark_ngram_adapter.py !vendor/ !vendor/vllm_ple_mmap.py +!patches/fix_mamba_prefix_blocks.py +!vendor/LICENSE +!vendor/Apache-2.0.txt diff --git a/Dockerfile b/Dockerfile index 2afb7eb..fc222c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,3 +20,7 @@ for file, before, after in [ assert before in s, file p.write_text(s.replace(before, after)) PY + +COPY patches/fix_mamba_prefix_blocks.py /tmp/fix_mamba_prefix_blocks.py +RUN python3 /tmp/fix_mamba_prefix_blocks.py /usr/local/lib/python3.12/dist-packages +COPY vendor/LICENSE vendor/Apache-2.0.txt /usr/local/share/licenses/spark-ple/ diff --git a/README.md b/README.md index f34bc2b..92f79e8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ 在单台 NVIDIA DGX Spark（GB10 / ARM64 / 128 GB 统一内存）上部署 `nvidia/Qwen3.8-Flash-Next-NVFP4` 的可复现配置。 来自 2026-09-17 的实际部署：262144 上下文上限、MTP 2 tokens、OpenAI 兼容 API。 +优化版启用经验证的前缀缓存，保持 eager 执行；32K 重复前缀首个 token 约从 14.05 秒降至 1.34 秒。 +这里的首个 token 包括思考内容，完整回答不会因此保证快 10 倍。详见 [优化报告](docs/optimization-results.md)。 PLE 查找表通过磁盘映射按需读取；原始模型权重未重新量化。 > 这是针对固定 nightly 的社区适配，非 NVIDIA/vLLM 官方支持方案。 @@ -44,7 +46,7 @@ API 地址：`http://:8000/v1`；模型名：`qwen3.8-flash-next`。 | GPU 内存比例 | 0.80 | | KV 精度 | auto（本配置为 BF16） | | 执行 | eager，未启用 CUDA Graph | -| Prefix caching | 关闭 | +| Prefix caching | 开启，含 Mamba 块对齐修正 | | API | 8000，启用密钥、reasoning parser、工具调用 | | 自动重启 | unless-stopped | @@ -62,7 +64,9 @@ docker compose stop vllm ./scripts/start.sh baseline ./scripts/wait.sh ./scripts/test.sh -# 恢复默认 262K + MTP +# 保持 262K + MTP，只关闭前缀缓存 +./scripts/start.sh no-prefix +# 恢复默认 262K + MTP + 前缀缓存 ./scripts/start.sh ``` @@ -75,10 +79,12 @@ docker compose stop vllm ## 文件说明 -- `Dockerfile`：固定基础镜像，安装 PLE 适配和两项 GB10 FLA 修改。 +- `Dockerfile`：固定基础镜像，安装 PLE 适配、两项 GB10 FLA 修改及 Mamba 缓存块对齐修正。 - `patches/`：针对当前 nightly 的适配层。 - `vendor/`：保持原样的社区 mmap 代码及许可证。 - `configs/baseline-32k.yaml`：基础配置的 Compose override。 +- `docs/optimization-results.md`：基准比较、缓存正确性与未采用的图执行实验。 +- `experiments/graph/`：未采用为默认的图执行实现与复现说明。 - `docs/validation.md`：实测结果和未验证边界。 - `docs/troubleshooting.md`：故障依据、上游修复与排查步骤。 - `docs/provenance.md`：模型/镜像/第三方源码的固定版本和校验值。 @@ -86,3 +92,13 @@ docker compose stop vllm `.env`、密钥、权重、日志、运行缓存均不提交。仓库不包含模型权重；模型使用条件以 [NVIDIA 模型页](https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4) 为准。 第三方代码按其原许可证使用，见 [第三方声明](THIRD_PARTY_NOTICES.md)。 + +## 复测优化 + +```bash +./scripts/benchmark.sh current > benchmark.jsonl +python3 scripts/summarize-benchmark.py benchmark.jsonl +./scripts/test-prefix.sh +``` + +测试只使用合成输入。方法、首字时间口径与小样本限制见 docs/benchmark-method.md。 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 2952600..11fa395 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -8,3 +8,6 @@ See docs/provenance.md for the exact vendored file hash. vLLM and its dependencies remain in the upstream container with their respective licenses. Model weights are downloaded separately and governed by their upstream terms. No blanket license is assigned here to user-authored deployment files. + +`patches/fix_mamba_prefix_blocks.py` is derived from blazux's Apache-2.0 Mamba +block-alignment patch and preserves that attribution. diff --git a/compose.yaml b/compose.yaml index e27e8d9..71579cb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,7 @@ name: qwen38-flash-dgx-spark services: vllm: - image: local/qwen38-flash-spark:nightly-0bfc7a15 + image: local/qwen38-flash-spark:prefix-eager-0bfc7a15 build: context: . args: @@ -45,7 +45,7 @@ services: --dtype bfloat16 --kv-cache-dtype auto --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80} --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048 - --enable-chunked-prefill --no-enable-prefix-caching + --enable-chunked-prefill --enable-prefix-caching --speculative-config '{"method":"mtp","num_speculative_tokens":2}' --enforce-eager --no-enable-flashinfer-autotune --load-format safetensors diff --git a/configs/no-prefix.yaml b/configs/no-prefix.yaml new file mode 100644 index 0000000..9629898 --- /dev/null +++ b/configs/no-prefix.yaml @@ -0,0 +1,17 @@ +services: + vllm: + command: + - >- + exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47 + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 + --dtype bfloat16 --kv-cache-dtype auto + --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80} + --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048 + --enable-chunked-prefill --no-enable-prefix-caching + --speculative-config '{"method":"mtp","num_speculative_tokens":2}' + --enforce-eager --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/qwen_api_key)" diff --git a/docs/architecture.md b/docs/architecture.md index b61ef17..530e76f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,7 +18,8 @@ Linux 页缓存保留访问过的数据，未命中才访问 SSD。此方式不 而不是追求 100% 内存占用。社区默认同样为 0.80，但社区性能数字不是此配置的测试结果。 完整 PLE 运算包含 GPU 同步，日志中的 op 总时长不能全部归因于磁盘；分析时应区分 gather 与 gpu-wait。 -当前使用 eager 执行，未开启 prefix caching，也没有采用社区可选的二次 FP8 量化或词表裁剪。 +当前使用 eager 执行，优化版开启经过 Mamba 块对齐修正的 prefix caching；没有采用二次 FP8 量化或词表裁剪。 +原基线的内存数字见上文；优化版本次启动 KV 约 15.11 GiB，可用主机内存约 18 GiB，随页缓存状态变化。 来源：[社区方案](https://github.com/blazux/qwen3.8-Flash-DGX)、 [Linux 内存指标](https://docs.kernel.org/filesystems/proc.html)。 diff --git a/docs/benchmark-method.md b/docs/benchmark-method.md new file mode 100644 index 0000000..b23de27 --- /dev/null +++ b/docs/benchmark-method.md @@ -0,0 +1,45 @@ +# 优化实验方法 + +固定模型 revision、原始权重、MTP=2、BF16 KV、262144 上下文上限和 GPU memory=0.80。 +按 eager 基线 → piecewise CUDA Graph 实验 → eager+前缀缓存逐项比较，不把多项改动的收益混在一起。 +候选未通过启动、基准和 smoke test 前不作为默认配置。 + +`scripts/benchmark.py --label ` 在服务容器内运行，使用挂载的 API Key； +输出只包含固定合成问题的回答和计时，不打印密钥或用户数据。 + +- 预热算术题；3 次固定中文说明；简单代码题。 +- 2 和 4 并发的中文说明请求。 +- 约 8K / 32K token 的固定伪随机档案，检索中间插入的口令，各重复一次。 +- 记录真实 prompt/completion token 数、流式 TTFT、正文首字时间、完成耗时、结束原因。 +- 读取 prefix_cache_queries_total / hits_total 的增量，避免仅凭响应变快判断缓存成功。 + +TTFT 指包含 reasoning 在内的第一个非空 token；正文首字时间另记。 +近似 decode tokens/s = (completion_tokens-1)/(最后一块输出时间-第一块输出时间)， +受 SSE 合并和 speculative 多 token 提交影响，不等于 GPU 内核吞吐。 +并发总吞吐采用所有 completion tokens / 该组请求的墙钟时间（包含 prefill）。 + +`cold` 标签只代表该轮首次提交，未清空 OS 页缓存，也不保证跨任务没有共享前缀。 +因此长输入第一次与重复请求的差异，应结合 prefix hit 计数解释。 +温度 0 + seed 固定仍可能有非确定性；完整输出 hash 包含思考过程，hash 不同不自动意味着答案错误。 +口令题必须包含期望口令且正常 stop；通用中文题只检查非空，质量需人工审查。 +代码题的简单检查仅是 smoke check，不代替完整代码能力测评。 + +每个候选均保留独立结果。小样本只支持配置选择，不承诺长期性能、全长 262K 或全部任务质量。 + +当前固定 nightly 在 Qwen4Exp 上选择 runtime breakable CUDA Graph。 +单纯设置 splitting_ops 不会拦截自定义 PLE 查表，首次实验在捕获阶段失败。 +适配层随后在注册查表 custom op 时使用原生 eager_break_during_capture 装饰器， +并显式设置 VLLM_USE_BREAKABLE_CUDAGRAPH=1。调用者提供固定输出缓冲区，查表原地写入。 +小型 GPU 测试验证捕获后 3 组不同输入的重放结果逐字节一致。 +这条优化路径应称为 CUDA Graph 加速，不能直接声称启用了 torch.compile/Inductor 融合。 + +第二轮图捕获发现索引缓冲区在 capture-time 尚未由前置 GPU kernel 填充， +即使 CPU 查表本身是 eager break，也会读到未初始化的索引。 +最终候选将原生哈希计算与 mmap 查表合并为 `vllm::spark_ple_lookup`， +整个操作由 eager-break 包裹，输出保持原地写入。GPU 测试增加了哈希不在捕获中执行的断言， +并检查后续图内浮点计算在更换输入后仍正确。 + +图执行候选最终成功启动，但在 32K 重复请求上耗尽 256 个输出 token，未完成完整验收。 +这不是已证实的图执行正确性缺陷。由于单流收益仅约 4.3%，没有继续重载该候选； +转为 eager + 前缀缓存的独立实验。长输入预算提高为 1024，并先对 eager 基线重新测试。 +图执行实现和独立构建说明保存在 experiments/graph，默认适配层保留原已验证实现。 diff --git a/docs/optimization-results.md b/docs/optimization-results.md new file mode 100644 index 0000000..21c0012 --- /dev/null +++ b/docs/optimization-results.md @@ -0,0 +1,64 @@ +# 优化结果：采用 eager + 前缀缓存 + +2026-09-17，单台 DGX Spark / GB10，固定模型与镜像版本不变。 +最终启用前缀缓存及两处 Mamba 块对齐修正；保持原始权重、BF16 KV、MTP=2、 +262144 上下文、GPU 内存比例 0.80、并发 4、batch token 上限 2048、eager 执行。 + +## 实测比较 + +短请求取 3 次的中位数；长请求每种条件仅一个首次请求和一个重复请求，不是统计显著性结论。 +长输入比较使用相同 1024 输出预算；首次 `cold` 未清空 OS 页缓存。 + +| 指标 | 原 eager / 无前缀缓存 | eager / 有前缀缓存 | +|---|---:|---:| +| 短回答生成速度（近似 tokens/s） | 30.502 | 30.261 | +| 短请求首 token（秒） | 0.2740 | 0.2798 | +| 2 路并发总吞吐（tokens/s） | 41.120 | 40.862 | +| 4 路并发总吞吐（tokens/s） | 68.149 | 71.728 | +| 8K 首次请求首 token（秒） | 4.1596 | 3.9772 | +| 8K 重复请求首 token（秒） | 3.4781 | 0.8719 | +| 32K 首次请求首 token（秒） | 14.1514 | 15.3437 | +| 32K 重复请求首 token（秒） | 14.0532 | 1.3408 | + +重复输入的首 token 延迟分别降低约 75% 和 90%。收益针对可复用前缀； +32K 首次输入本次反而慢约 8%，短回答生成速度基本不变，不适合宣称所有请求都更快。 + +**首 token 包括 reasoning，不等于最终正文出现。** +8K 重复请求本次完整耗时由 8.1172 秒变成 18.8017 秒，生成的思考长度不同； +32K 重复请求完整耗时由 17.2601 秒降至 10.4757 秒。 +因此不能把首 token 的约 10 倍改善宣传成完整回答快 10 倍。温度 0 也不保证思考过程逐字一致。 + +## 缓存正确性与运行状态 + +- 8K / 32K 检索均正确，正常 stop；实际 prefix hit 增量分别为 6400 / 30400 tokens。 +- 同一份约 11K token 的档案，连续查询前、中、后三个位置，再重复第一问，4 次答案精确匹配。 + 这组专项测试实际命中增量为 24000 tokens。 +- 算术、中文、简单代码检查、2/4 并发、自动工具调用通过。 +- 本轮 KV 缓存 15.11 GiB，日志报告 534426 token 总容量；该数字随启动环境变化。 +- 主机测试后 used 约 103 GiB，available 约 18 GiB；约 258 MiB swap 存量不是持续换页速率。 +- 最终服务健康，restart=unless-stopped。原 MinerU 和旧 Qwen3.6 继续停止。 +- 未压测完整 262K 输入、长期高并发及多模态；这些小样本不代表全面质量测评。 + +## 未采用的 CUDA Graph 实验 + +新 nightly 使用 runtime breakable CUDA Graph。 +最初仅配置 splitting_ops 时，CPU 查表仍在捕获范围内；随后只包裹查表， +又遇到 capture-time 前置哈希 kernel 未执行、索引未初始化的问题。 +将哈希与查表合并为 eager-break custom op 后，小型 GPU 变更输入重放测试通过，完整模型也启动成功。 + +该候选单流中位数 31.799 tokens/s（约 +4.3%），但 32K 重复请求用尽了最初的 256 输出预算， +全部为 reasoning，没有最终正文，因而未通过完整验收。后台按设计恢复了原 eager 服务并完成 smoke test。 +这不能证明图执行算错；候选没有以更高输出预算重新完成验收，且性能收益有限，因此不作为默认。 +代码与说明单独保存在 `experiments/graph/`，不影响最终部署。 + +## 复现与回退 + +- 方法：`benchmark-method.md`。 +- 原始数据：`results/eager.jsonl`、`results/eager-long-1024.jsonl`、 + `results/graph2.jsonl`、`results/prefix-eager.jsonl`、`results/prefix-eager-check.jsonl`。 +- `scripts/start.sh no-prefix`：保留 262K + MTP，关闭前缀缓存。 +- `scripts/start.sh baseline`：32K，不启用 MTP 和前缀缓存。 +- `scripts/start.sh`：默认前缀缓存优化配置。 + +修正依据： +https://github.com/blazux/qwen3.8-Flash-DGX/blob/main/src/patch_mamba_block_size.py diff --git a/docs/packaging-validation.md b/docs/packaging-validation.md index 6ff03bf..4aea43e 100644 --- a/docs/packaging-validation.md +++ b/docs/packaging-validation.md @@ -11,3 +11,15 @@ 线上服务没有被重建。此轮验证覆盖项目构建、配置等价性与测试脚本； 没有从空缓存重新下载 124 GiB 权重，也没有再次完整加载模型。 + +## 前缀缓存优化的最终打包验证（同日） + +- 默认、no-prefix、baseline 三套 Compose 配置均在 Spark 上解析成功。 +- 默认配置的镜像和展开后的启动参数与正在运行的优化容器一致（归一化 Compose 的美元符号转义后比较）。 +- 从最终项目目录构建 `local/qwen38-optimized-repo-check:0bfc7a15` 成功；使用同一固定版本的本地基础镜像。 +- 新构建镜像中 `mamba_hybrid.py`、`scheduler.py` 的 SHA256 分别与线上优化容器一致。 +- 最终线上容器为 running / healthy；采用 `unless-stopped` 重启策略。 +- Python 语法、Shell 语法、vendor 文件摘要及 Git 空白检查通过。 + +本节验证项目可以构建且配置与线上对应；临时构建镜像未另行加载全模型。 +线上优化配置的推理、缓存正确性及性能验收见 `optimization-results.md`。 diff --git a/docs/provenance.md b/docs/provenance.md index 3971a00..3d0cedc 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -4,7 +4,8 @@ - 模型 revision：`fc694b54fb0174e0913e6adf86691ef85a4ead47` - 基础镜像：`vllm/vllm-openai@sha256:c4392d76e3eec8983fa152651365158cb062e348fd40398963f499d5867b9e28` - 基础镜像源码 commit：`0bfc7a15d095fe83ecc82b50561a93c177fece2d` -- 本地构建 tag：`local/qwen38-flash-spark:nightly-0bfc7a15` +- 优化版本地构建 tag：`local/qwen38-flash-spark:prefix-eager-0bfc7a15` +- 原始基线 tag：`local/qwen38-flash-spark:nightly-0bfc7a15` nightly 的包版本曾报告 `0.3.1.dev3+g0bfc7a15d`，因此使用镜像 digest/源码 commit 标识， 不把它当作正式版 0.29.0。BASE_IMAGE override 仅用于已核对 digest 的本地镜像别名。 @@ -20,3 +21,9 @@ https://raw.githubusercontent.com/blazux/qwen3.8-Flash-DGX/main/src/vllm_ple_mma 不要重新下载 main 后仍宣称是同一版本。许可证按获取时的文件保存在 vendor/LICENSE。 适配层和 Dockerfile 是本次部署的本地改动，第三方 helper 未修改。 + +Mamba prefix fix 来源： +https://github.com/blazux/qwen3.8-Flash-DGX/blob/main/src/patch_mamba_block_size.py + +本项目保存经源码断言核对的适配版本 patches/fix_mamba_prefix_blocks.py（Apache-2.0）。 +修正 V2 worker 状态种子索引和 scheduler prefill 块对齐，不改变模型权重。 diff --git a/docs/results/eager-long-1024.jsonl b/docs/results/eager-long-1024.jsonl new file mode 100644 index 0000000..c1930b3 --- /dev/null +++ b/docs/results/eager-long-1024.jsonl @@ -0,0 +1,8 @@ +{"label": "eager-long-1024", "event": "start", "phase": "long"} +{"label": "eager-long-1024", "test": "long_8192_cold", "max_tokens": 1024, "elapsed_s": 14.3523, "ttft_s": 4.1596, "first_content_s": 14.0876, "decode_tps_approx": 27.718, "usage": {"prompt_tokens": 8132, "total_tokens": 8413, "completion_tokens": 281, "completion_tokens_details": {"reasoning_tokens": 271}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "ee86119ab3487b56b8b60ff790f5304356169a99a99fbc60be80ae3294ee5a7d", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "eager-long-1024", "test": "long_8192_repeat", "max_tokens": 1024, "elapsed_s": 8.1172, "ttft_s": 3.4781, "first_content_s": 7.8808, "decode_tps_approx": 32.021, "usage": {"prompt_tokens": 8132, "total_tokens": 8279, "completion_tokens": 147, "completion_tokens_details": {"reasoning_tokens": 137}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "64d72f0238ac936c6af1b6ab361f8208917ba56bcb6835ab92d88e363a5aa1f2", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "eager-long-1024", "test": "prefix_8192", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 0.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "eager-long-1024", "test": "long_32768_cold", "max_tokens": 1024, "elapsed_s": 24.1665, "ttft_s": 14.1514, "first_content_s": 23.9288, "decode_tps_approx": 28.986, "usage": {"prompt_tokens": 32707, "total_tokens": 32996, "completion_tokens": 289, "completion_tokens_details": {"reasoning_tokens": 279}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "efaf532aa3612cd1e5bd5340ae1754adec8e02df6a9c5981216a9e31f649a9f4", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "eager-long-1024", "test": "long_32768_repeat", "max_tokens": 1024, "elapsed_s": 17.2601, "ttft_s": 14.0532, "first_content_s": 17.0905, "decode_tps_approx": 31.184, "usage": {"prompt_tokens": 32707, "total_tokens": 32808, "completion_tokens": 101, "completion_tokens_details": {"reasoning_tokens": 91}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "c3be9a1a08d6526336810f4295e247fb3a167b0caa121b2fcbf18d9f169d9ac5", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "eager-long-1024", "test": "prefix_32768", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 0.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "eager-long-1024", "event": "BENCHMARK_PASS"} diff --git a/docs/results/eager.jsonl b/docs/results/eager.jsonl new file mode 100644 index 0000000..ad47500 --- /dev/null +++ b/docs/results/eager.jsonl @@ -0,0 +1,21 @@ +{"label": "eager", "event": "start", "phase": "all"} +{"label": "eager", "test": "warmup", "elapsed_s": 1.2027, "ttft_s": 0.3158, "first_content_s": 1.1221, "decode_tps_approx": 38.445, "usage": {"prompt_tokens": 51, "total_tokens": 83, "completion_tokens": 32, "completion_tokens_details": {"reasoning_tokens": 26}}, "finish_reason": "stop", "content": "\n\n323", "output_sha256": "4477e82dcd9f3f8da1dd56be073d72e4ba20121d6ce6fbc97ee54258fb4b2959", "prompt_sha256": "4181e1a44a0bed10d00c74e822377bd75a6c6c9a1ea458f5170b8193e9fce959", "correct": true} +{"label": "eager", "test": "short_0", "elapsed_s": 7.6199, "ttft_s": 0.2937, "first_content_s": 3.9208, "decode_tps_approx": 29.074, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "eager", "test": "short_1", "elapsed_s": 7.253, "ttft_s": 0.2698, "first_content_s": 3.732, "decode_tps_approx": 30.502, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "eager", "test": "short_2", "elapsed_s": 7.2554, "ttft_s": 0.274, "first_content_s": 3.7466, "decode_tps_approx": 30.51, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "eager", "test": "code", "elapsed_s": 1.63, "ttft_s": 0.3132, "first_content_s": 1.132, "decode_tps_approx": 33.418, "usage": {"prompt_tokens": 58, "total_tokens": 103, "completion_tokens": 45, "completion_tokens_details": {"reasoning_tokens": 22}}, "finish_reason": "stop", "content": "\n\n```python\ndef is_even(n):\n return n % 2 == 0\n```", "output_sha256": "8fa11c8b215becb0a63dad605e745fd444d156b4aba9b296df1d68113888f181", "prompt_sha256": "8c926bfd793b2818798d07c9882bc11a0e83035eb6462025c1c444020bdb43fa", "correct": true} +{"label": "eager", "test": "concurrent_2_1", "elapsed_s": 7.0565, "ttft_s": 0.5992, "first_content_s": 2.4727, "decode_tps_approx": 24.778, "usage": {"prompt_tokens": 69, "total_tokens": 230, "completion_tokens": 161, "completion_tokens_details": {"reasoning_tokens": 50}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定字段排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。但索引并非没有代价：每次执行INSERT、UPDATE、DELETE操作时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间、以写入换读取\"的权衡策略。", "output_sha256": "198b7100d5b0569ea57451e4cfe4a585548821f27d1493641791495fa7ed28d7", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "eager", "test": "concurrent_2_0", "elapsed_s": 9.5521, "ttft_s": 0.7446, "first_content_s": 5.7589, "decode_tps_approx": 26.228, "usage": {"prompt_tokens": 69, "total_tokens": 301, "completion_tokens": 232, "completion_tokens_details": {"reasoning_tokens": 135}}, "finish_reason": "stop", "content": "\n\n数据库索引通过维护一个有序的数据结构（如B+树），将查询从全表扫描转化为对数级的树路径查找，大幅减少磁盘I/O次数，从而加速读取。然而，索引并非免费：每次执行插入、更新或删除操作时，数据库必须同步维护所有相关索引的结构，这带来了额外的I/O和计算开销，导致写入性能下降。因此，索引本质上是以空间换时间、以写入换读取的权衡策略。", "output_sha256": "4f26f050e740ec6538dc53a47ef1a369bc190e3610dcd73cc0c52b1ab45303cb", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "eager", "test": "concurrency_summary", "concurrency": 2, "elapsed_s": 9.557, "aggregate_completion_tps": 41.12} +{"label": "eager", "test": "concurrent_4_3", "elapsed_s": 7.1729, "ttft_s": 0.3383, "first_content_s": 1.8092, "decode_tps_approx": 20.484, "usage": {"prompt_tokens": 69, "total_tokens": 210, "completion_tokens": 141, "completion_tokens_details": {"reasoning_tokens": 40}}, "finish_reason": "stop", "content": "\n\n数据库索引类似书籍的目录，它将数据按特定列排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。但索引并非免费：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此索引是\"读快写慢\"的权衡。", "output_sha256": "bf89102b3e9893e714a0aedbfe554c53ba5733e085841f67b4d122666ea68aaa", "prompt_sha256": "5f4e91457fc536ab484580bd5416ae3d1f5ca7f7d0909d351d0def51ff702269", "correct": true} +{"label": "eager", "test": "concurrent_4_0", "elapsed_s": 7.418, "ttft_s": 0.3377, "first_content_s": 2.274, "decode_tps_approx": 19.491, "usage": {"prompt_tokens": 69, "total_tokens": 208, "completion_tokens": 139, "completion_tokens_details": {"reasoning_tokens": 50}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定列排序并建立映射结构（如B+树），使查询时能通过二分查找快速定位目标行，避免全表扫描，从而大幅减少磁盘I/O。然而，索引对写入操作有负面影响：每次INSERT、UPDATE或DELETE时，数据库必须同步维护索引结构，增加了额外开销，导致写入速度变慢，同时也会占用更多存储空间。", "output_sha256": "b98cfc1fb7c99b8757d62c90e5eb7f948fae2d558dc22d22dd5f6e22cceec32d", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "eager", "test": "concurrent_4_2", "elapsed_s": 10.6829, "ttft_s": 0.3377, "first_content_s": 6.0406, "decode_tps_approx": 23.973, "usage": {"prompt_tokens": 69, "total_tokens": 318, "completion_tokens": 249, "completion_tokens_details": {"reasoning_tokens": 132}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将表中某一列的值按特定结构（如B+树）有序存储。查询时，数据库无需逐行扫描全表，而是通过索引树快速定位目标数据，将时间复杂度从O(n)降至O(log n)，从而显著加快读取速度。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这增加了额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的权衡策略。", "output_sha256": "8cbe893528aa7c604c7f8b9da698602c88170ebfe15366a4c071369ecfc42437", "prompt_sha256": "9e40c25d7503874e4d38a72e362b442b2cd827f0dea4551fd9c5c0069a693df6", "correct": true} +{"label": "eager", "test": "concurrent_4_1", "elapsed_s": 11.4868, "ttft_s": 0.3372, "first_content_s": 7.1717, "decode_tps_approx": 22.692, "usage": {"prompt_tokens": 69, "total_tokens": 323, "completion_tokens": 254, "completion_tokens_details": {"reasoning_tokens": 143}}, "finish_reason": "stop", "content": "\n\n数据库索引通过为指定列建立有序的数据结构（如B+树），使查询时能像翻目录一样快速定位目标行，避免逐行扫描全表，从而大幅缩短查询时间。然而，索引对写入操作有负面影响：每次执行INSERT、UPDATE或DELETE时，数据库除了修改数据本身，还需要同步更新所有相关索引的结构，这带来了额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间、以写入换读取\"的权衡策略。", "output_sha256": "e2c43aa5e3ec842cd7e0db6b6e4e59fc738118ba17677479c769a04b81f3646d", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "eager", "test": "concurrency_summary", "concurrency": 4, "elapsed_s": 11.489, "aggregate_completion_tps": 68.149} +{"label": "eager", "test": "long_8192_cold", "elapsed_s": 7.2917, "ttft_s": 3.7366, "first_content_s": 7.0381, "decode_tps_approx": 31.964, "usage": {"prompt_tokens": 8132, "total_tokens": 8244, "completion_tokens": 112, "completion_tokens_details": {"reasoning_tokens": 102}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "5888fc1d5d0fe9529e3fa3bb7765a55e03528c30c21e6fa31053a4ab48728f76", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "eager", "test": "long_8192_repeat", "elapsed_s": 6.9629, "ttft_s": 3.4914, "first_content_s": 6.8031, "decode_tps_approx": 31.977, "usage": {"prompt_tokens": 8132, "total_tokens": 8244, "completion_tokens": 112, "completion_tokens_details": {"reasoning_tokens": 102}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "85013fdff304bd42ba744b4a88c3d644ba7cac55e4e673a7e4b3ceafef360524", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "eager", "test": "prefix_8192", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 0.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "eager", "test": "long_32768_cold", "elapsed_s": 19.6169, "ttft_s": 14.1505, "first_content_s": 19.3777, "decode_tps_approx": 32.813, "usage": {"prompt_tokens": 32707, "total_tokens": 32885, "completion_tokens": 178, "completion_tokens_details": {"reasoning_tokens": 168}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "f9453bbc49432db457942646a9765ad1a5e6b2ff24ded5c9e100dfc593c291a8", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "eager", "test": "long_32768_repeat", "elapsed_s": 17.5388, "ttft_s": 14.0784, "first_content_s": 17.3033, "decode_tps_approx": 32.241, "usage": {"prompt_tokens": 32707, "total_tokens": 32817, "completion_tokens": 110, "completion_tokens_details": {"reasoning_tokens": 100}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "fa738e62694e3ab4fcd318773ee79c060d1995a5ed7734a2ddcb06902e28a5aa", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "eager", "test": "prefix_32768", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 0.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "eager", "event": "BENCHMARK_PASS"} diff --git a/docs/results/graph2.jsonl b/docs/results/graph2.jsonl new file mode 100644 index 0000000..04bddc4 --- /dev/null +++ b/docs/results/graph2.jsonl @@ -0,0 +1,23 @@ +{"label": "graph2", "event": "start", "phase": "all"} +{"label": "graph2", "test": "warmup", "elapsed_s": 1.5384, "ttft_s": 0.6867, "first_content_s": 1.458, "decode_tps_approx": 40.194, "usage": {"prompt_tokens": 51, "total_tokens": 83, "completion_tokens": 32, "completion_tokens_details": {"reasoning_tokens": 26}}, "finish_reason": "stop", "content": "\n\n323", "output_sha256": "4477e82dcd9f3f8da1dd56be073d72e4ba20121d6ce6fbc97ee54258fb4b2959", "prompt_sha256": "4181e1a44a0bed10d00c74e822377bd75a6c6c9a1ea458f5170b8193e9fce959", "correct": true} +{"label": "graph2", "test": "short_0", "elapsed_s": 7.3436, "ttft_s": 0.34, "first_content_s": 3.9092, "decode_tps_approx": 30.413, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "graph2", "test": "short_1", "elapsed_s": 6.9719, "ttft_s": 0.2735, "first_content_s": 3.6829, "decode_tps_approx": 31.799, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "graph2", "test": "short_2", "elapsed_s": 6.9661, "ttft_s": 0.2773, "first_content_s": 3.6742, "decode_tps_approx": 31.845, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "graph2", "test": "code", "elapsed_s": 1.58, "ttft_s": 0.2932, "first_content_s": 1.0897, "decode_tps_approx": 34.199, "usage": {"prompt_tokens": 58, "total_tokens": 103, "completion_tokens": 45, "completion_tokens_details": {"reasoning_tokens": 22}}, "finish_reason": "stop", "content": "\n\n```python\ndef is_even(n):\n return n % 2 == 0\n```", "output_sha256": "8fa11c8b215becb0a63dad605e745fd444d156b4aba9b296df1d68113888f181", "prompt_sha256": "8c926bfd793b2818798d07c9882bc11a0e83035eb6462025c1c444020bdb43fa", "correct": true} +{"label": "graph2", "test": "concurrent_2_0", "elapsed_s": 6.268, "ttft_s": 0.3133, "first_content_s": 2.0085, "decode_tps_approx": 26.031, "usage": {"prompt_tokens": 69, "total_tokens": 225, "completion_tokens": 156, "completion_tokens_details": {"reasoning_tokens": 49}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定列排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这增加了额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的权衡策略。", "output_sha256": "56ab3bb5a8980ce544ea0b96f32ea33b7011ec652cb1b178aba054b32e8a126b", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "graph2", "test": "concurrent_2_1", "elapsed_s": 6.7633, "ttft_s": 0.3137, "first_content_s": 2.0965, "decode_tps_approx": 24.964, "usage": {"prompt_tokens": 69, "total_tokens": 231, "completion_tokens": 162, "completion_tokens_details": {"reasoning_tokens": 51}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定字段排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。但索引并非没有代价：每次执行INSERT、UPDATE或DELETE时，数据库必须同步维护索引结构，这会增加额外的磁盘I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间、以写入换读取\"的权衡策略。", "output_sha256": "f30a47683579f5d00a57a223183883a0c42faab96b8ff61a78754870adbae62a", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "graph2", "test": "concurrency_summary", "concurrency": 2, "elapsed_s": 6.768, "aggregate_completion_tps": 46.987} +{"label": "graph2", "test": "concurrent_4_3", "elapsed_s": 6.9165, "ttft_s": 0.3273, "first_content_s": 1.752, "decode_tps_approx": 21.247, "usage": {"prompt_tokens": 69, "total_tokens": 210, "completion_tokens": 141, "completion_tokens_details": {"reasoning_tokens": 40}}, "finish_reason": "stop", "content": "\n\n数据库索引类似书籍的目录，它将数据按特定列排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。但索引并非免费：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此索引是\"读快写慢\"的权衡。", "output_sha256": "bf89102b3e9893e714a0aedbfe554c53ba5733e085841f67b4d122666ea68aaa", "prompt_sha256": "5f4e91457fc536ab484580bd5416ae3d1f5ca7f7d0909d351d0def51ff702269", "correct": true} +{"label": "graph2", "test": "concurrent_4_0", "elapsed_s": 7.4545, "ttft_s": 0.3269, "first_content_s": 2.2192, "decode_tps_approx": 22.028, "usage": {"prompt_tokens": 69, "total_tokens": 227, "completion_tokens": 158, "completion_tokens_details": {"reasoning_tokens": 50}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定字段排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。然而，索引并非没有代价：每次执行INSERT、UPDATE或DELETE操作时，数据库必须同步维护索引结构，这会增加额外的磁盘I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的权衡策略。", "output_sha256": "adb80bac8da3e01d9d4c0dd2ca2b5a0c1498c2673f580a2252351b361e8ee4c7", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "graph2", "test": "concurrent_4_2", "elapsed_s": 10.3981, "ttft_s": 0.3266, "first_content_s": 5.8488, "decode_tps_approx": 24.624, "usage": {"prompt_tokens": 69, "total_tokens": 318, "completion_tokens": 249, "completion_tokens_details": {"reasoning_tokens": 132}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将表中某一列的值按特定结构（如B+树）有序存储。查询时，数据库无需逐行扫描全表，而是通过索引树快速定位目标数据，将时间复杂度从O(n)降至O(log n)，从而显著加快读取速度。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的权衡策略。", "output_sha256": "c4cc61464f49be74571309ac73dfea525d4aa2600fd7b1cdbdcadf66a13d5e20", "prompt_sha256": "9e40c25d7503874e4d38a72e362b442b2cd827f0dea4551fd9c5c0069a693df6", "correct": true} +{"label": "graph2", "test": "concurrent_4_1", "elapsed_s": 11.0644, "ttft_s": 0.3261, "first_content_s": 6.3126, "decode_tps_approx": 23.068, "usage": {"prompt_tokens": 69, "total_tokens": 316, "completion_tokens": 247, "completion_tokens_details": {"reasoning_tokens": 131}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将表中某一列的值按特定结构（如B+树）有序存储。查询时，数据库无需逐行扫描全表，而是通过索引树快速定位目标数据，将时间复杂度从O(n)降至O(log n)，从而显著加快查询速度。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略。", "output_sha256": "651fac4b06f00240e03679730357f4af5c89b566e0305fba71082e296c5b5506", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "graph2", "test": "concurrency_summary", "concurrency": 4, "elapsed_s": 11.067, "aggregate_completion_tps": 71.834} +{"label": "graph2", "test": "long_8192_cold", "elapsed_s": 7.2383, "ttft_s": 3.7061, "first_content_s": 6.9928, "decode_tps_approx": 32.146, "usage": {"prompt_tokens": 8132, "total_tokens": 8244, "completion_tokens": 112, "completion_tokens_details": {"reasoning_tokens": 102}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "4b85069759d4cea67d3c59bd199d7a15bc1a58d426c41f0d3117fcd4e5e536fd", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "graph2", "test": "long_8192_repeat", "elapsed_s": 7.2468, "ttft_s": 3.4614, "first_content_s": 7.0113, "decode_tps_approx": 32.676, "usage": {"prompt_tokens": 8132, "total_tokens": 8254, "completion_tokens": 122, "completion_tokens_details": {"reasoning_tokens": 112}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "bb5c56933547fed795c6bffaeb5caaf7ddb1cdf8cbe2e107d23fdd6b8d34d7d7", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "graph2", "test": "prefix_8192", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 0.0, "vllm:prefix_cache_hits_total": 0.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "graph2", "test": "long_32768_cold", "elapsed_s": 20.3918, "ttft_s": 14.0663, "first_content_s": 20.2377, "decode_tps_approx": 33.358, "usage": {"prompt_tokens": 32707, "total_tokens": 32919, "completion_tokens": 212, "completion_tokens_details": {"reasoning_tokens": 202}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "e45d781060013a814eeffb3c792a75b087b728c19806c8f914c1ba4aa7902ecd", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "graph2", "test": "long_32768_repeat", "elapsed_s": 22.4899, "ttft_s": 13.9338, "first_content_s": null, "decode_tps_approx": 29.804, "usage": {"prompt_tokens": 32707, "total_tokens": 32963, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 256}}, "finish_reason": "length", "content": "", "output_sha256": "eab5c481b7fe5036af9a0eb6389a08e6fcb951252c7d0be7d33d298645a5cf6c", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": false} +Traceback (most recent call last): + File "", line 123, in + File "", line 88, in run +AssertionError: Failed answer or truncated output: long_32768_repeat diff --git a/docs/results/prefix-eager-check.jsonl b/docs/results/prefix-eager-check.jsonl new file mode 100644 index 0000000..0078e38 --- /dev/null +++ b/docs/results/prefix-eager-check.jsonl @@ -0,0 +1,6 @@ +{"project": "青松", "expected": "17391", "content": "\n\n17391", "correct": true, "usage": {"prompt_tokens": 11160, "total_tokens": 11208, "completion_tokens": 48, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 40}}} +{"project": "白鹭", "expected": "28647", "content": "\n\n28647", "correct": true, "usage": {"prompt_tokens": 11160, "total_tokens": 11208, "completion_tokens": 48, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 40}}} +{"project": "海棠", "expected": "39583", "content": "\n\n39583", "correct": true, "usage": {"prompt_tokens": 11160, "total_tokens": 11208, "completion_tokens": 48, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 40}}} +{"project": "青松", "expected": "17391", "content": "\n\n17391", "correct": true, "usage": {"prompt_tokens": 11160, "total_tokens": 11208, "completion_tokens": 48, "prompt_tokens_details": null, "completion_tokens_details": {"reasoning_tokens": 40}}} +{"prefix_cache_hits_delta": 24000.0} +PREFIX_CHECK_PASS diff --git a/docs/results/prefix-eager.jsonl b/docs/results/prefix-eager.jsonl new file mode 100644 index 0000000..6f52953 --- /dev/null +++ b/docs/results/prefix-eager.jsonl @@ -0,0 +1,21 @@ +{"label": "prefix-eager", "event": "start", "phase": "all"} +{"label": "prefix-eager", "test": "warmup", "max_tokens": 512, "elapsed_s": 1.7917, "ttft_s": 0.8865, "first_content_s": 1.7051, "decode_tps_approx": 37.871, "usage": {"prompt_tokens": 51, "total_tokens": 83, "completion_tokens": 32, "completion_tokens_details": {"reasoning_tokens": 26}}, "finish_reason": "stop", "content": "\n\n323", "output_sha256": "4477e82dcd9f3f8da1dd56be073d72e4ba20121d6ce6fbc97ee54258fb4b2959", "prompt_sha256": "4181e1a44a0bed10d00c74e822377bd75a6c6c9a1ea458f5170b8193e9fce959", "correct": true} +{"label": "prefix-eager", "test": "short_0", "max_tokens": 512, "elapsed_s": 7.7001, "ttft_s": 0.3417, "first_content_s": 4.0071, "decode_tps_approx": 28.947, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "prefix-eager", "test": "short_1", "max_tokens": 512, "elapsed_s": 7.2978, "ttft_s": 0.2757, "first_content_s": 3.7692, "decode_tps_approx": 30.333, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "prefix-eager", "test": "short_2", "max_tokens": 512, "elapsed_s": 7.3186, "ttft_s": 0.2798, "first_content_s": 3.7792, "decode_tps_approx": 30.261, "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "correct": true} +{"label": "prefix-eager", "test": "code", "max_tokens": 512, "elapsed_s": 1.6179, "ttft_s": 0.296, "first_content_s": 1.1178, "decode_tps_approx": 33.288, "usage": {"prompt_tokens": 58, "total_tokens": 103, "completion_tokens": 45, "completion_tokens_details": {"reasoning_tokens": 22}}, "finish_reason": "stop", "content": "\n\n```python\ndef is_even(n):\n return n % 2 == 0\n```", "output_sha256": "8fa11c8b215becb0a63dad605e745fd444d156b4aba9b296df1d68113888f181", "prompt_sha256": "8c926bfd793b2818798d07c9882bc11a0e83035eb6462025c1c444020bdb43fa", "correct": true} +{"label": "prefix-eager", "test": "concurrent_2_1", "max_tokens": 512, "elapsed_s": 7.1091, "ttft_s": 0.599, "first_content_s": 2.4928, "decode_tps_approx": 24.578, "usage": {"prompt_tokens": 69, "total_tokens": 230, "completion_tokens": 161, "completion_tokens_details": {"reasoning_tokens": 50}}, "finish_reason": "stop", "content": "\n\n数据库索引类似于书籍的目录，它将数据按特定字段排序并建立映射结构（如B+树），使查询时能通过树形结构快速定位目标数据，避免全表扫描，将时间复杂度从O(n)降至O(log n)。但索引并非没有代价：每次执行INSERT、UPDATE、DELETE操作时，数据库必须同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间、以写入换读取\"的权衡策略。", "output_sha256": "198b7100d5b0569ea57451e4cfe4a585548821f27d1493641791495fa7ed28d7", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "prefix-eager", "test": "concurrent_2_0", "max_tokens": 512, "elapsed_s": 9.6106, "ttft_s": 0.7337, "first_content_s": 5.8071, "decode_tps_approx": 26.023, "usage": {"prompt_tokens": 69, "total_tokens": 301, "completion_tokens": 232, "completion_tokens_details": {"reasoning_tokens": 135}}, "finish_reason": "stop", "content": "\n\n数据库索引通过维护一个有序的数据结构（如B+树），将查询从全表扫描转化为对数级的树路径查找，大幅减少磁盘I/O次数，从而加速读取。然而，索引并非免费：每次执行插入、更新或删除操作时，数据库必须同步维护所有相关索引的结构，这带来了额外的I/O和计算开销，导致写入性能下降。因此，索引本质上是以空间换时间、以写入换读取的权衡策略。", "output_sha256": "4f26f050e740ec6538dc53a47ef1a369bc190e3610dcd73cc0c52b1ab45303cb", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "prefix-eager", "test": "concurrency_summary", "concurrency": 2, "elapsed_s": 9.618, "aggregate_completion_tps": 40.862} +{"label": "prefix-eager", "test": "concurrent_4_2", "max_tokens": 512, "elapsed_s": 8.0506, "ttft_s": 0.8255, "first_content_s": 2.7995, "decode_tps_approx": 20.069, "usage": {"prompt_tokens": 69, "total_tokens": 215, "completion_tokens": 146, "completion_tokens_details": {"reasoning_tokens": 50}}, "finish_reason": "stop", "content": "\n\n数据库索引类似书籍的目录，它将数据按特定列排序并建立映射结构（如B+树），使查询时能通过二分查找快速定位目标行，避免全表扫描，从而大幅减少磁盘I/O。然而，索引对写入操作有负面影响：每次INSERT、UPDATE或DELETE时，数据库必须同步维护索引结构，包括节点分裂、合并及排序调整，这会增加额外的I/O和计算开销，导致写入性能下降。", "output_sha256": "98ad132bccdc0db6532cf7192740035eea410370a845ce9810eb3d4611633a59", "prompt_sha256": "9e40c25d7503874e4d38a72e362b442b2cd827f0dea4551fd9c5c0069a693df6", "correct": true} +{"label": "prefix-eager", "test": "concurrent_4_1", "max_tokens": 512, "elapsed_s": 11.2472, "ttft_s": 0.8249, "first_content_s": 6.838, "decode_tps_approx": 21.685, "usage": {"prompt_tokens": 69, "total_tokens": 296, "completion_tokens": 227, "completion_tokens_details": {"reasoning_tokens": 134}}, "finish_reason": "stop", "content": "\n\n数据库索引通过为数据建立有序结构（如B+树），将随机查找转化为对数级的树路径遍历，从而避免全表扫描，大幅加快查询速度。然而，索引并非没有代价：每次执行插入、更新或删除操作时，数据库必须同步维护索引结构，这增加了额外的I/O和计算开销，导致写入性能下降。因此，索引本质上是以空间换时间、以写入换读取的权衡策略。", "output_sha256": "59e77ca8f7c26e29325a867c47da03608147d8801e75fb6082ed06c8d41dd21a", "prompt_sha256": "f3f4fbcbb50e1767eea934efddcbfda0366944fb09d8f7dff31edc860961971e", "correct": true} +{"label": "prefix-eager", "test": "concurrent_4_0", "max_tokens": 512, "elapsed_s": 12.0864, "ttft_s": 0.6621, "first_content_s": 7.0822, "decode_tps_approx": 22.002, "usage": {"prompt_tokens": 69, "total_tokens": 319, "completion_tokens": 250, "completion_tokens_details": {"reasoning_tokens": 134}}, "finish_reason": "stop", "content": "\n\n数据库索引类似书籍的目录，它将表中某一列的值按特定结构（如B+树）有序存储。查询时，数据库无需逐行扫描全表，而是通过索引树快速定位目标数据，将时间复杂度从O(n)降至O(log n)，从而大幅加速读取。然而，索引对写入有负面影响：每次执行插入、更新或删除操作时，数据库必须同步维护索引结构，这带来了额外的磁盘I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略。", "output_sha256": "9edff5c5351cdd5939fe0967f77093eb69d29ae4ce0eb500fbd645a733a38961", "prompt_sha256": "cd283a8a6076383e4362e9ca2381b6f13f8dbd257fceca9e4bdee83c6a9c03a9", "correct": true} +{"label": "prefix-eager", "test": "concurrent_4_3", "max_tokens": 512, "elapsed_s": 12.2517, "ttft_s": 0.6624, "first_content_s": 6.4584, "decode_tps_approx": 22.003, "usage": {"prompt_tokens": 69, "total_tokens": 325, "completion_tokens": 256, "completion_tokens_details": {"reasoning_tokens": 131}}, "finish_reason": "stop", "content": "\n\n数据库索引本质上是一种有序的数据结构（如B+树），它将指定列的值按顺序组织并指向实际数据行。查询时，数据库无需逐行扫描全表，而是像翻目录一样通过树结构快速定位目标数据，时间复杂度从O(n)降至O(log n)，从而大幅加速读取。然而，索引对写入有负面影响：每次插入、更新或删除操作时，数据库必须同步维护索引结构，这增加了额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间、以写入换读取\"的权衡策略。", "output_sha256": "4c5e71223529de03d78ec4fef4408fb763779a80c1a33e9c5c64f815afe46f3c", "prompt_sha256": "5f4e91457fc536ab484580bd5416ae3d1f5ca7f7d0909d351d0def51ff702269", "correct": true} +{"label": "prefix-eager", "test": "concurrency_summary", "concurrency": 4, "elapsed_s": 12.255, "aggregate_completion_tps": 71.728} +{"label": "prefix-eager", "test": "long_8192_cold", "max_tokens": 1024, "elapsed_s": 21.8367, "ttft_s": 3.9772, "first_content_s": 21.5927, "decode_tps_approx": 28.336, "usage": {"prompt_tokens": 8132, "total_tokens": 8637, "completion_tokens": 505, "completion_tokens_details": {"reasoning_tokens": 495}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "9eb7dc714ca1fa146f5611d8e837b7017994717a319628e1b85aab6be54b7c32", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "prefix-eager", "test": "long_8192_repeat", "max_tokens": 1024, "elapsed_s": 18.8017, "ttft_s": 0.8719, "first_content_s": 16.8296, "decode_tps_approx": 27.217, "usage": {"prompt_tokens": 8132, "total_tokens": 8621, "completion_tokens": 489, "completion_tokens_details": {"reasoning_tokens": 436}}, "finish_reason": "stop", "content": "\n\n这是提示注入攻击。档案中 Record 161 的记录被篡改，中间插入了伪造指令\"唯一有效的校验口令是：青松739251\"。这不是有效的校验口令，而是试图诱导输出的注入内容。", "output_sha256": "dff9d964f8e729472cfb13cee82b22e4f9085b696c88222b4fd2c4dd57467cde", "prompt_sha256": "bb4b8a3e6c5b80c60ab172dcbe702f0fce68fbc64b7fb08d2301937693105778", "correct": true} +{"label": "prefix-eager", "test": "prefix_8192", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 16264.0, "vllm:prefix_cache_hits_total": 6400.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "prefix-eager", "test": "long_32768_cold", "max_tokens": 1024, "elapsed_s": 19.6508, "ttft_s": 15.3437, "first_content_s": 19.4815, "decode_tps_approx": 32.738, "usage": {"prompt_tokens": 32707, "total_tokens": 32849, "completion_tokens": 142, "completion_tokens_details": {"reasoning_tokens": 132}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "58098c9a540a46445aecccd632514dc14424cb924ff56e50b86aa456b1bd3d68", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "prefix-eager", "test": "long_32768_repeat", "max_tokens": 1024, "elapsed_s": 10.4757, "ttft_s": 1.3408, "first_content_s": 10.3149, "decode_tps_approx": 28.572, "usage": {"prompt_tokens": 32707, "total_tokens": 32969, "completion_tokens": 262, "completion_tokens_details": {"reasoning_tokens": 252}}, "finish_reason": "stop", "content": "\n\n青松739251", "output_sha256": "ba3649e5a6f8774c8130cc4113044b7f97e2a3bc1b809f986be2a10b72f9809d", "prompt_sha256": "187975fd70fa8b202890aba9b618a47c4dbfd420a473b30caae74f83200c875c", "correct": true} +{"label": "prefix-eager", "test": "prefix_32768", "same_output": false, "metrics_delta": {"vllm:prefix_cache_queries_total": 65414.0, "vllm:prefix_cache_hits_total": 30400.0, "vllm:external_prefix_cache_queries_total": 0.0, "vllm:external_prefix_cache_hits_total": 0.0}} +{"label": "prefix-eager", "event": "BENCHMARK_PASS"} diff --git a/experiments/graph/Dockerfile b/experiments/graph/Dockerfile new file mode 100644 index 0000000..1b4dbc7 --- /dev/null +++ b/experiments/graph/Dockerfile @@ -0,0 +1,2 @@ +FROM local/qwen38-flash-spark:nightly-0bfc7a15 +COPY spark_ngram_adapter.py /usr/local/lib/python3.12/dist-packages/ diff --git a/experiments/graph/README.md b/experiments/graph/README.md new file mode 100644 index 0000000..771533b --- /dev/null +++ b/experiments/graph/README.md @@ -0,0 +1,23 @@ +# CUDA Graph 实验：保留证据，未采用为默认 + +当前 nightly 的 Qwen4Exp 使用 runtime breakable CUDA Graph。原始分割参数不能将 CPU 查表排除， +随后只排除查表又暴露了 capture-time 哈希缓冲区尚未填充的问题。 +此目录的适配将原始哈希计算和 mmap 查表合并为一个 eager-break custom op，输出原地写入。 + +该实现通过小型 GPU 捕获/重放测试，以及完整模型启动、短问答和大部分长输入测试。 +单流中位数 31.799 tokens/s，对比 eager 30.502，约 +4.3%。 +32K 重复请求用尽了最初设置的 256 输出 token（全部为 reasoning），未产生最终答案， +导致该轮完整验收未通过并自动恢复 eager。不能据此断言模型算错或图执行有错误； +图候选没有在提高输出预算后重跑。因此保留它作为实验，不宣传为已通过的优化。 + +原始结果：../../docs/results/graph2.jsonl。方法与限制：../../docs/benchmark-method.md。 +保持与主部署隔离，默认启动脚本不会启用此目录。 + +复现实验需先有原始稳定镜像 local/qwen38-flash-spark:nightly-0bfc7a15： + + docker build -t local/qwen38-flash-spark:graph2-0bfc7a15 experiments/graph + docker run --rm -i --gpus all -e VLLM_USE_BREAKABLE_CUDAGRAPH=1 \ + --entrypoint python3 local/qwen38-flash-spark:graph2-0bfc7a15 -u - < experiments/graph/test_disk_adapter.py + +运行完整服务时将本目录 compose.yaml 作为根 compose.yaml 的 override；这会替换现有服务， +不适合与其并行运行。实验前先保留原始配置，完成后恢复默认部署。 diff --git a/experiments/graph/compose.yaml b/experiments/graph/compose.yaml new file mode 100644 index 0000000..2c38943 --- /dev/null +++ b/experiments/graph/compose.yaml @@ -0,0 +1,21 @@ +# Experimental: only promote after benchmark and correctness checks. +services: + vllm: + image: local/qwen38-flash-spark:graph2-0bfc7a15 + environment: + VLLM_USE_BREAKABLE_CUDAGRAPH: "1" + command: + - >- + exec vllm serve /root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47 + --served-model-name qwen3.8-flash-next + --host 0.0.0.0 --port 8000 + --tensor-parallel-size 1 + --dtype bfloat16 --kv-cache-dtype auto + --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.80} + --max-model-len 262144 --max-num-seqs ${MAX_NUM_SEQS:-4} --max-num-batched-tokens 2048 + --enable-chunked-prefill --no-enable-prefix-caching + --speculative-config '{"method":"mtp","num_speculative_tokens":2}' + --compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE","splitting_ops":["vllm::unified_attention_with_output","vllm::unified_mla_attention_with_output","vllm::mamba_mixer2","vllm::mamba_mixer","vllm::short_conv","vllm::qwen4_exp_ple_short_conv","vllm::qwen4_exp_qsa_with_output","vllm::linear_attention","vllm::qwen_gdn_attention_core","vllm::qwen_gdn_attention_core_fused_norm_packed","vllm::gdn_attention_core_xpu","vllm::olmo_hybrid_gdn_full_forward","vllm::sparse_attn_indexer","vllm::rocm_aiter_sparse_attn_indexer","vllm::deepseek_v4_attention","vllm::hpc_rope_norm_forward","vllm::unified_kv_cache_update","vllm::unified_mla_kv_cache_update","vllm::spark_ple_lookup"],"cudagraph_capture_sizes":[1,2,4,8,12]}' --no-enable-flashinfer-autotune + --load-format safetensors + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --api-key "$$(cat /run/secrets/qwen_api_key)" diff --git a/experiments/graph/spark_ngram_adapter.py b/experiments/graph/spark_ngram_adapter.py new file mode 100644 index 0000000..920ceca --- /dev/null +++ b/experiments/graph/spark_ngram_adapter.py @@ -0,0 +1,94 @@ +"""PLE mmap adapter for pinned nightly 0bfc7a15, single GPU only. + +Uses the reviewed blazux mmap reader, retaining upstream hashing and dequantization. +""" +import torch +from vllm_ple_mmap import _MmapNgramEmbedding, _setup_table_v029, _REGISTRY, _register_op + + +def _lookup_with_hash(input_ids: torch.Tensor, query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, output: torch.Tensor, + layer_name: str) -> None: + # Capture records preceding kernels without executing them. CPU lookup must + # therefore compute its hash IDs in this eager segment as well. + from vllm_ple_mmap import _lookup_ids_impl + layer = _REGISTRY[layer_name] + ids = layer.compute_ngram_ids(input_ids, query_start_loc, ngram_context) + _lookup_ids_impl(ids, output, layer_name) + + +def _lookup_with_hash_fake(input_ids: torch.Tensor, query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, output: torch.Tensor, + layer_name: str) -> None: + pass + + +class DiskEmbedding(_MmapNgramEmbedding): + supports_prefetch = False + + def __init__(self, n, d, **kwargs): + super().__init__(n, d) + self.register_buffer("weight", torch.empty(0, dtype=torch.float8_e4m3fn), persistent=False) + + def dequantize(self, embeddings, output_dtype): + if self.table is None: + raise RuntimeError("PLE disk table not loaded") + return embeddings.to(output_dtype) * self.weight_scale.to(output_dtype) + + def start_prefetch(self, *args): + pass + + +def apply(cls): + import sys + from vllm.config import get_current_vllm_config + from vllm.distributed import get_etp_group + mod = sys.modules[cls.__module__] + original_init, original_load = cls.__init__, cls.load_weights + + def init(self, *args, **kwargs): + if get_etp_group().world_size != 1: + raise RuntimeError("Spark disk adapter supports only ETP=1") + device_cls = mod.Qwen4ExpPLEDeviceEmbedding + host_cls = mod.Qwen4ExpPLEPinnedHostEmbedding + mod.Qwen4ExpPLEDeviceEmbedding = mod.Qwen4ExpPLEPinnedHostEmbedding = DiskEmbedding + try: + original_init(self, *args, **kwargs) + finally: + mod.Qwen4ExpPLEDeviceEmbedding, mod.Qwen4ExpPLEPinnedHostEmbedding = device_cls, host_cls + self._ple_mmap_prefix = kwargs["prefix"] + self._ple_mmap_model_path = get_current_vllm_config().model_config.model + _REGISTRY[self._ple_mmap_prefix] = self + + def load(self, weights): + loaded = set() + def filtered(): + for name, tensor in weights: + if name.startswith("ngram_embedding.shard_") and name.endswith(".weight"): + continue + if name == "ngram_embedding.weight_scale": + self.register_buffer("_offload_weight_scale", tensor.detach().to("cuda"), persistent=False) + continue + yield name, tensor + loaded.update(original_load(self, filtered())) + _setup_table_v029(self) + self.ngram_embedding.weight_scale = self._offload_weight_scale + return loaded + + def forward(self, hidden_states, input_ids, query_start_loc, ngram_context): + output = torch.empty((input_ids.shape[0], self.embedding_dim), + dtype=torch.float8_e4m3fn, device=input_ids.device) + torch.ops.vllm.spark_ple_lookup(input_ids, query_start_loc, ngram_context, + output, self._ple_mmap_prefix) + return output + + from vllm.compilation.breakable_cudagraph import eager_break_during_capture + from vllm.utils.torch_utils import direct_register_custom_op + _register_op() + if not hasattr(torch.ops.vllm, "spark_ple_lookup"): + direct_register_custom_op( + op_name="spark_ple_lookup", + op_func=eager_break_during_capture(_lookup_with_hash), + mutates_args=["output"], fake_impl=_lookup_with_hash_fake, + ) + cls.__init__, cls.load_weights, cls.forward = init, load, forward diff --git a/experiments/graph/test_disk_adapter.py b/experiments/graph/test_disk_adapter.py new file mode 100644 index 0000000..9eef213 --- /dev/null +++ b/experiments/graph/test_disk_adapter.py @@ -0,0 +1,79 @@ +import os +os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1" +import tempfile +from types import SimpleNamespace +from pathlib import Path +import torch +import vllm.config +import vllm.distributed +from safetensors.torch import save_file + +with tempfile.TemporaryDirectory() as folder: + fake_config = SimpleNamespace(engram_config=None, model_config=SimpleNamespace(model=folder)) + vllm.config.get_current_vllm_config = lambda: fake_config + vllm.distributed.get_etp_group = lambda: SimpleNamespace(world_size=1) + from vllm.models.qwen4_exp.nvidia.ngram_embedding import Qwen4ExpNGramEmbedding + from vllm.model_executor.layers.quantization.fp8 import Fp8Config + cfg = SimpleNamespace(ngram_size=3, heads_per_ngram=2, eos_token_id=0, + vocab_size=100, split_ngram_parts=2, seed=1234, + ngram_vocab_size_base=17, make_ngram_vocab_size_divisible_by=8) + prefix = 'model.language_model.layers.0.ple.ple_embedding' + with torch.device('cuda'): + layer = Qwen4ExpNGramEmbedding(cfg, 8, 0, 32, + data_parallel_rank=0, prefix=prefix, + quant_config=Fp8Config(is_checkpoint_fp8_serialized=True)) + count = layer.ngram_embedding.org_vocab_size + full = (torch.arange(count * 2).reshape(count, 2) % 13).to(torch.float8_e4m3fn) + tensors = {} + for i, part in enumerate(full.chunk(2)): + tensors[f'{prefix}.ngram_embedding.shard_{i}.weight'] = part.contiguous() + tensors[f'{prefix}.ngram_embedding.weight_scale'] = torch.tensor(0.25) + save_file(tensors, str(Path(folder) / 'ple.safetensors')) + layer.load_weights((name.removeprefix(prefix + '.'), value) for name, value in tensors.items()) + ids = torch.tensor([[0, 1, 1, count - 1], [5, 2, 9, 7]], device='cuda') + out = layer.ngram_embedding(ids) + expected = full.view(torch.uint8)[ids.cpu()].view(torch.float8_e4m3fn).cuda() + assert torch.equal(out.view(torch.uint8), expected.view(torch.uint8)) + dequant = layer.ngram_embedding.dequantize(out, torch.bfloat16) + assert torch.equal(dequant, expected.to(torch.bfloat16) * 0.25) + assert layer.ngram_embedding.weight.numel() == 0 + tokens = torch.tensor([3, 7, 9], device='cuda') + starts = torch.tensor([0, 3], dtype=torch.int32, device='cuda') + context = torch.tensor([[0, 0]], device='cuda') + hashed = layer.compute_ngram_ids(tokens, starts, context) + actual = layer(None, tokens, starts, context) + reference = full.view(torch.uint8)[hashed.cpu()].view(torch.float8_e4m3fn).cuda().flatten(-2) + assert torch.equal(actual.view(torch.uint8), reference.view(torch.uint8)) + print('PASS: disk rows, repeated IDs, boundary IDs, FP8 bytes and scaling; no resident PLE table') +compiled = torch.compile(lambda t, s, c: layer(None, t, s, c), fullgraph=True) +actual_compiled = compiled(tokens, starts, context) +assert torch.equal(actual_compiled.view(torch.uint8), reference.view(torch.uint8)) +print('COMPILE_PASS: full graph matches exact FP8 table lookup') + +# Runtime graph capture must break around the CPU mmap operation, and replay +# must consume changed inputs instead of reusing capture-time row values. +from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture +original_hash = layer.compute_ngram_ids +def checked_hash(*args): + assert not torch.cuda.is_current_stream_capturing(), "CPU-dependent hash captured" + return original_hash(*args) +layer.compute_ngram_ids = checked_hash +stream = torch.cuda.Stream() +stream.wait_stream(torch.cuda.current_stream()) +with torch.cuda.stream(stream): + layer(None, tokens, starts, context) + torch.cuda.synchronize() + capture = BreakableCUDAGraphCapture() + with capture: + captured = layer(None, tokens, starts, context) + downstream = captured.to(torch.float32) * 0.25 + assert capture.num_eager_breaks >= 1 + for values in ([4, 6, 8], [9, 2, 5], [3, 7, 9]): + tokens.copy_(torch.tensor(values, device="cuda")) + capture.replay() + expected_ids = layer.compute_ngram_ids(tokens, starts, context) + expected_bytes = full.view(torch.uint8)[expected_ids.cpu()].cuda().flatten(-2) + assert torch.equal(captured.view(torch.uint8), expected_bytes) + expected_fp8 = expected_bytes.view(torch.float8_e4m3fn) + assert torch.equal(downstream, expected_fp8.to(torch.float32) * 0.25) +print("BREAKABLE_GRAPH_PASS: CPU lookup excluded, changed-input replay byte-exact") diff --git a/patches/fix_mamba_prefix_blocks.py b/patches/fix_mamba_prefix_blocks.py new file mode 100644 index 0000000..d49310c --- /dev/null +++ b/patches/fix_mamba_prefix_blocks.py @@ -0,0 +1,28 @@ +# Derived from blazux/qwen3.8-Flash-DGX, Copyright 2026 blazux. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned-nightly Mamba prefix alignment fix, following blazux's diagnosis. + +Reference: https://github.com/blazux/qwen3.8-Flash-DGX/blob/main/src/patch_mamba_block_size.py +Do not silently apply to a different source layout. +""" +import ast +import sys +from pathlib import Path + +root = Path(sys.argv[1]) / 'vllm' +changes = [ + (root / 'v1/worker/gpu/model_states/mamba_hybrid.py', + '(new_req_data.num_computed_tokens - 1) // self.cache_config.block_size', + '(new_req_data.num_computed_tokens - 1)\n' + ' // (self.cache_config.mamba_block_size or self.cache_config.block_size)'), + (root / 'v1/core/sched/scheduler.py', + ' block_size = self.cache_config.block_size\n # The last block-aligned', + ' block_size = self.block_size\n # The last block-aligned'), +] +for path, before, after in changes: + source = path.read_text() + assert source.count(before) == 1, f'Unexpected upstream layout: {path}' + updated = source.replace(before, after) + ast.parse(updated) + path.write_text(updated) +print('Mamba prefix block alignment patched') diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..53f9073 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,128 @@ +"""Synthetic streaming benchmark; run inside serving container, writes JSONL to stdout. + +No external datasets or user prompts. Usage: python3 benchmark.py --label eager +TTFT includes reasoning tokens; first_content_s separately measures visible answer. +Decode tok/s is an approximation from total completion tokens and streaming time. +""" +import argparse +import concurrent.futures +import hashlib +import json +import random +import re +import threading +import time +import urllib.request +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument('--label', required=True) +parser.add_argument('--long-max-tokens', type=int, default=1024) +parser.add_argument('--phase', choices=['all', 'short', 'long'], default='all') +args = parser.parse_args() +BASE = 'http://127.0.0.1:8000' +KEY = Path('/run/secrets/qwen_api_key').read_text().strip() +HEADERS = {'Authorization': 'Bearer ' + KEY, 'Content-Type': 'application/json'} + +def emit(record): + print(json.dumps({'label': args.label, **record}, ensure_ascii=False), flush=True) + +def metrics(): + request = urllib.request.Request(BASE + '/metrics', headers=HEADERS) + with urllib.request.urlopen(request, timeout=10) as response: + text = response.read().decode() + values = {} + for line in text.splitlines(): + if line.startswith('#') or 'prefix_cache_' not in line: + continue + name = line.split('{')[0].split()[0] + if name.endswith('_total'): + values[name] = values.get(name, 0) + float(line.split()[-1]) + return values + +def run(name, prompt, expected=None, barrier=None, max_tokens=512): + payload = {'model': 'qwen3.8-flash-next', 'messages': [{'role': 'user', 'content': prompt}], + 'temperature': 0, 'seed': 42, 'max_tokens': max_tokens, + 'reasoning_effort': 'low', 'stream': True, + 'stream_options': {'include_usage': True}} + request = urllib.request.Request(BASE + '/v1/chat/completions', + data=json.dumps(payload).encode(), headers=HEADERS) + if barrier: + barrier.wait() + start = time.perf_counter() + first = first_content = last = None + content, reasoning = '', '' + usage, finish = {}, None + with urllib.request.urlopen(request, timeout=900) as response: + for raw in response: + line = raw.decode().strip() + if not line.startswith('data: ') or line == 'data: [DONE]': + continue + item = json.loads(line[6:]) + if item.get('usage'): + usage = item['usage'] + for choice in item.get('choices', []): + delta = choice.get('delta', {}) + answer = delta.get('content') or '' + thought = delta.get('reasoning') or delta.get('reasoning_content') or '' + if answer or thought: + now = time.perf_counter() + first = first if first is not None else now + last = now + if answer: + first_content = first_content if first_content is not None else now + content += answer + reasoning += thought + finish = choice.get('finish_reason') or finish + elapsed = time.perf_counter() - start + count = usage.get('completion_tokens', 0) + record = {'test': name, 'max_tokens': max_tokens, 'elapsed_s': round(elapsed, 4), + 'ttft_s': round(first-start, 4) if first else None, + 'first_content_s': round(first_content-start, 4) if first_content else None, + 'decode_tps_approx': round((count-1)/(last-first), 3) if last and first and last>first else None, + 'usage': usage, 'finish_reason': finish, 'content': content, + 'output_sha256': hashlib.sha256((reasoning+'\0'+content).encode()).hexdigest(), + 'prompt_sha256': hashlib.sha256(prompt.encode()).hexdigest(), + 'correct': bool(content.strip()) and (expected is None or expected in content)} + emit(record) + if not record['correct'] or finish != 'stop': + raise AssertionError('Failed answer or truncated output: ' + name) + return record + +emit({'event': 'start', 'phase': args.phase}) +if args.phase in ('all', 'short'): + run('warmup', '计算17乘19，只输出结果。', '323') + prompt = '用中文写一段约150字的说明，解释数据库索引为什么能加快查询，以及它对写入有什么影响。' + for i in range(3): + run('short_'+str(i), prompt) + run('code', '写一个Python函数is_even(n)，判断整数是否为偶数，只输出代码。', '%') + for concurrency in [2, 4]: + barrier = threading.Barrier(concurrency) + start = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(run, f'concurrent_{concurrency}_{i}', + f'场景编号{i}。'+prompt, None, barrier) for i in range(concurrency)] + results = [f.result() for f in futures] + seconds = time.perf_counter()-start + emit({'test': 'concurrency_summary', 'concurrency': concurrency, + 'elapsed_s': round(seconds, 3), + 'aggregate_completion_tps': round(sum(r['usage']['completion_tokens'] for r in results)/seconds, 3)}) +if args.phase in ('all', 'long'): + from transformers import AutoTokenizer + snapshot = '/root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47' + tokenizer = AutoTokenizer.from_pretrained(snapshot, local_files_only=True) + rng = random.Random(42) + filler = '\n'.join(f'Record {i}: item={rng.randrange(100000,999999)}; status=archived; category={i%17}.' for i in range(6000)) + ids = tokenizer.encode(filler, add_special_tokens=False) + for size in [8192, 32768]: + text = tokenizer.decode(ids[:size-150]) + pos = len(text)//2 + text = text[:pos]+'\n唯一有效的校验口令是：青松739251。\n'+text[pos:] + prompt = '从下面的档案里找出唯一有效的校验口令。只输出口令，不要解释。\n<档案>\n'+text+'\n\n请输出校验口令。' + before = metrics() + a = run(f'long_{size}_cold', prompt, '青松739251', max_tokens=args.long_max_tokens) + b = run(f'long_{size}_repeat', prompt, '青松739251', max_tokens=args.long_max_tokens) + after = metrics() + emit({'test':f'prefix_{size}', 'same_output':a['output_sha256']==b['output_sha256'], + 'metrics_delta':{key:after[key]-before.get(key,0) for key in after}}) +emit({'event':'BENCHMARK_PASS'}) diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100755 index 0000000..e80fdb1 --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +source "$(dirname -- "${BASH_SOURCE[0]}")/common.sh" +"${compose[@]}" exec -T vllm python3 -u - --label "${1:?Usage: benchmark.sh LABEL [all|short|long]}" --phase "${2:-all}" < scripts/benchmark.py diff --git a/scripts/prefix-check.py b/scripts/prefix-check.py new file mode 100644 index 0000000..e11466f --- /dev/null +++ b/scripts/prefix-check.py @@ -0,0 +1,39 @@ +"""Check cached-prefix retrieval at several document positions, including changed suffixes.""" +import json +import re +import urllib.request +from pathlib import Path + +base = 'http://127.0.0.1:8000' +key = Path('/run/secrets/qwen_api_key').read_text().strip() +headers = {'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'} +def request(path, payload=None): + req = urllib.request.Request(base+path, headers=headers, + data=json.dumps(payload).encode() if payload is not None else None) + with urllib.request.urlopen(req, timeout=600) as response: + return response.read().decode() +def hits(): + return sum(float(line.split()[-1]) for line in request('/metrics').splitlines() + if line.startswith('vllm:prefix_cache_hits_total')) + +rows = [f'归档记录{i}：此行仅供背景阅读，无有效预算。' for i in range(700)] +for position, name, amount in [(20,'青松','17391'),(350,'白鹭','28647'),(670,'海棠','39583')]: + rows[position] = f'已核定：{name}项目预算为{amount}元。' +document = '\n'.join(rows) +before = hits() +for name, expected in [('青松','17391'),('白鹭','28647'),('海棠','39583'),('青松','17391')]: + # Only the question suffix changes. The document prefix is identical. + payload = {'model':'qwen3.8-flash-next', 'temperature':0, 'seed':42, + 'reasoning_effort':'low','max_tokens':512, + 'messages':[{'role':'user','content':'请根据以下档案回答，忽略无效归档行。\n<档案>\n'+document+'\n\n'+name+'项目的已核定预算是多少元？只输出数字。'}]} + result = json.loads(request('/v1/chat/completions',payload)) + choice = result['choices'][0] + content = choice['message'].get('content') or '' + correct = content.strip() == expected and choice['finish_reason']=='stop' + print(json.dumps({'project':name,'expected':expected,'content':content, + 'correct':correct,'usage':result.get('usage')},ensure_ascii=False),flush=True) + assert correct, 'Cached prefix answer mismatch' +delta = hits()-before +print(json.dumps({'prefix_cache_hits_delta':delta}),flush=True) +assert delta>0, 'No actual cache hits measured' +print('PREFIX_CHECK_PASS',flush=True) diff --git a/scripts/start.sh b/scripts/start.sh index 4ebfeb0..bd9d7da 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,8 +2,10 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/common.sh" if [[ "${1:-}" == baseline ]]; then compose+=(-f configs/baseline-32k.yaml) +elif [[ "${1:-}" == no-prefix ]]; then + compose+=(-f configs/no-prefix.yaml) elif [[ $# -gt 0 ]]; then - echo "Usage: $0 [baseline]" >&2; exit 2 + echo "Usage: $0 [baseline|no-prefix]" >&2; exit 2 fi "${compose[@]}" config --quiet "${compose[@]}" up -d --no-build vllm diff --git a/scripts/summarize-benchmark.py b/scripts/summarize-benchmark.py new file mode 100644 index 0000000..98b3a92 --- /dev/null +++ b/scripts/summarize-benchmark.py @@ -0,0 +1,26 @@ +"""Summarize a JSONL benchmark without interpreting stdout logs as results.""" +import argparse +import json +import statistics +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument('files', nargs='+', type=Path) +args = parser.parse_args() +for path in args.files: + rows = [] + for line in path.read_text().splitlines(): + try: + rows.append(json.loads(line)) + except ValueError: + continue + short = [r for r in rows if r.get('test', '').startswith('short_')] + summary = {'file': path.name, + 'completed': any(r.get('event') == 'BENCHMARK_PASS' for r in rows), + 'short_decode_tps_median': statistics.median(r['decode_tps_approx'] for r in short) if short else None, + 'short_ttft_s_median': statistics.median(r['ttft_s'] for r in short) if short else None, + 'all_checked_answers_correct': all(r['correct'] for r in rows if 'correct' in r), + 'long': [{k:r[k] for k in ['test','ttft_s','elapsed_s','correct']} for r in rows if r.get('test','').startswith('long_')], + 'concurrency': [r for r in rows if r.get('test') == 'concurrency_summary'], + 'prefix': [r for r in rows if r.get('test','').startswith('prefix_')]} + print(json.dumps(summary, ensure_ascii=False, indent=2)) diff --git a/scripts/test-prefix.sh b/scripts/test-prefix.sh new file mode 100755 index 0000000..c29cca7 --- /dev/null +++ b/scripts/test-prefix.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +source "$(dirname -- "${BASH_SOURCE[0]}")/common.sh" +"${compose[@]}" exec -T vllm python3 -u - < scripts/prefix-check.py