21 lines
1.5 KiB
Python
21 lines
1.5 KiB
Python
"""Summarize tuning receipts without omitting failed variants."""
|
|
import argparse,json,pathlib,re,statistics
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=pathlib.Path);a=p.parse_args();out={}
|
|
for folder in sorted(a.root.glob('*/*')):
|
|
if not folder.is_dir():continue
|
|
entry={};status=folder/'status.json'
|
|
if status.exists():entry.update(json.loads(status.read_text()))
|
|
for f in ['screen.jsonl','quality-medium.jsonl']:
|
|
path=folder/f
|
|
if not path.exists():continue
|
|
rows=[json.loads(l) for l in path.read_text().splitlines() if l]
|
|
entry[f]={'count':len(rows),'failed':[r['name'] for r in rows if not r['passed']]}
|
|
if f=='screen.jsonl':
|
|
entry['decode_tps']={tag:statistics.median(r['decode_tps'] for r in rows if r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps')) for tag in ['code','chinese','reasoning'] if any(r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps') for r in rows)}
|
|
entry['latency']={r['name']:{'ttft_s':r.get('ttft_s'),'elapsed_s':r.get('elapsed_s'),'passed':r['passed']} for r in rows if r['name'] in ['long-context-0','long-context-1','long-context-image','video-120s','multi-image']}
|
|
log=folder/'server.log'
|
|
if log.exists():
|
|
s=log.read_text();entry['memory_log']=[l for l in s.splitlines() if any(t in l for t in ['Model loading took','Available KV cache memory','GPU KV cache size','To serve at least one request'])]
|
|
if entry:out[str(folder.relative_to(a.root))]=entry
|
|
print(json.dumps(out,ensure_ascii=False,indent=2))
|