27 lines
1.3 KiB
Python
27 lines
1.3 KiB
Python
"""Summarize a JSONL benchmark without interpreting stdout logs as results."""
|
|
import argparse
|
|
import json
|
|
import statistics
|
|
from pathlib import Path
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('files', nargs='+', type=Path)
|
|
args = parser.parse_args()
|
|
for path in args.files:
|
|
rows = []
|
|
for line in path.read_text().splitlines():
|
|
try:
|
|
rows.append(json.loads(line))
|
|
except ValueError:
|
|
continue
|
|
short = [r for r in rows if r.get('test', '').startswith('short_')]
|
|
summary = {'file': path.name,
|
|
'completed': any(r.get('event') == 'BENCHMARK_PASS' for r in rows),
|
|
'short_decode_tps_median': statistics.median(r['decode_tps_approx'] for r in short) if short else None,
|
|
'short_ttft_s_median': statistics.median(r['ttft_s'] for r in short) if short else None,
|
|
'all_checked_answers_correct': all(r['correct'] for r in rows if 'correct' in r),
|
|
'long': [{k:r[k] for k in ['test','ttft_s','elapsed_s','correct']} for r in rows if r.get('test','').startswith('long_')],
|
|
'concurrency': [r for r in rows if r.get('test') == 'concurrency_summary'],
|
|
'prefix': [r for r in rows if r.get('test','').startswith('prefix_')]}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|