32 lines
2.8 KiB
Python
32 lines
2.8 KiB
Python
"""Summarize complete/partial receipts without dropping failures."""
|
|
import argparse,json,statistics
|
|
from pathlib import Path
|
|
p=argparse.ArgumentParser();p.add_argument('directory',type=Path);a=p.parse_args();result={}
|
|
for variant in ['baseline','mtp2']:
|
|
path=a.directory/(variant+'.jsonl')
|
|
if not path.exists():continue
|
|
rows=[json.loads(l) for l in path.read_text().splitlines() if l];summary={'requests':len(rows),'failures':[{'name':x['name'],'error':x.get('error'),'content':x.get('content')} for x in rows if not x['passed']],'timing':{}}
|
|
for tag in ['code','chinese','reasoning']:
|
|
samples=[r for r in rows if r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps')]
|
|
if samples:summary['timing'][tag]={'samples':len(samples),'decode_tps_median':statistics.median(r['decode_tps'] for r in samples),'ttft_s_median':statistics.median(r['ttft_s'] for r in samples),'elapsed_s_median':statistics.median(r['elapsed_s'] for r in samples),'reasoning_tokens':[r['usage'].get('completion_tokens_details',{}).get('reasoning_tokens') for r in samples]}
|
|
soak=[r for r in rows if r['name'].startswith('soak-')];summary['soak']={'requests':len(soak),'sum_request_seconds':sum(r.get('elapsed_s',0) for r in soak),'failures':sum(not r['passed'] for r in soak)}
|
|
long=[r for r in rows if r['name']=='long-output-16384'];summary['long_output']=[{k:v for k,v in r.items() if k not in ('content','reasoning')} for r in long]
|
|
quality=a.directory/(variant+'-quality.jsonl')
|
|
if quality.exists():
|
|
q=[json.loads(l) for l in quality.read_text().splitlines()];summary['quality']={'tasks':len(q),'failures':[r['name'] for r in q if not r['passed']]}
|
|
result[variant]=summary
|
|
if all(v in result for v in ['baseline','mtp2']):
|
|
result['speedup']={k:result['mtp2']['timing'][k]['decode_tps_median']/v['decode_tps_median'] for k,v in result['baseline']['timing'].items() if k in result['mtp2']['timing']}
|
|
for variant in ['baseline','mtp2']:
|
|
path=a.directory/(variant+'-quality-medium.jsonl')
|
|
if path.exists() and variant in result:
|
|
q=[json.loads(l) for l in path.read_text().splitlines()];result[variant]['quality_medium']={'tasks':len(q),'failures':[r['name'] for r in q if not r['passed']]}
|
|
rpath=a.directory/'resources.jsonl'
|
|
if rpath.exists():
|
|
rows=[json.loads(l) for f in [rpath,a.directory/'resources-resumed.jsonl'] if f.exists() for l in f.read_text().splitlines() if l];gpus=[]
|
|
for r in rows:
|
|
try:gpus.append([float(x.strip()) for x in r['gpu'].split(',')])
|
|
except (ValueError,KeyError):pass
|
|
result['resources']={'samples':len(rows),'min_host_available_gib':min(r['available_gib'] for r in rows),'oom_kill_max':max(r['oom_kill'] for r in rows),'safeguard_triggered':any(r['stop'] for r in rows),'gpu_memory_peak_mib':max((g[0] for g in gpus),default=None),'gpu_temperature_peak_c':max((g[3] for g in gpus),default=None)}
|
|
print(json.dumps(result,ensure_ascii=False,indent=2))
|