Enable validated prefix caching and record DGX Spark optimization benchmarks

2026-09-17 22:09:58 +08:00
parent 5f3030260e
commit 1e2f48d1a4
28 changed files with 707 additions and 8 deletions
@@ -0,0 +1,26 @@
+"""Summarize a JSONL benchmark without interpreting stdout logs as results."""
+import argparse
+import json
+import statistics
+from pathlib import Path
+
+parser = argparse.ArgumentParser()
+parser.add_argument('files', nargs='+', type=Path)
+args = parser.parse_args()
+for path in args.files:
+    rows = []
+    for line in path.read_text().splitlines():
+        try:
+            rows.append(json.loads(line))
+        except ValueError:
+            continue
+    short = [r for r in rows if r.get('test', '').startswith('short_')]
+    summary = {'file': path.name,
+               'completed': any(r.get('event') == 'BENCHMARK_PASS' for r in rows),
+               'short_decode_tps_median': statistics.median(r['decode_tps_approx'] for r in short) if short else None,
+               'short_ttft_s_median': statistics.median(r['ttft_s'] for r in short) if short else None,
+               'all_checked_answers_correct': all(r['correct'] for r in rows if 'correct' in r),
+               'long': [{k:r[k] for k in ['test','ttft_s','elapsed_s','correct']} for r in rows if r.get('test','').startswith('long_')],
+               'concurrency': [r for r in rows if r.get('test') == 'concurrency_summary'],
+               'prefix': [r for r in rows if r.get('test','').startswith('prefix_')]}
+    print(json.dumps(summary, ensure_ascii=False, indent=2))