"""Summarize complete/partial receipts without dropping failures.""" import argparse,json,statistics from pathlib import Path p=argparse.ArgumentParser();p.add_argument('directory',type=Path);a=p.parse_args();result={} for variant in ['baseline','mtp2']: path=a.directory/(variant+'.jsonl') if not path.exists():continue rows=[json.loads(l) for l in path.read_text().splitlines() if l];summary={'requests':len(rows),'failures':[{'name':x['name'],'error':x.get('error'),'content':x.get('content')} for x in rows if not x['passed']],'timing':{}} for tag in ['code','chinese','reasoning']: samples=[r for r in rows if r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps')] if samples:summary['timing'][tag]={'samples':len(samples),'decode_tps_median':statistics.median(r['decode_tps'] for r in samples),'ttft_s_median':statistics.median(r['ttft_s'] for r in samples),'elapsed_s_median':statistics.median(r['elapsed_s'] for r in samples),'reasoning_tokens':[r['usage'].get('completion_tokens_details',{}).get('reasoning_tokens') for r in samples]} soak=[r for r in rows if r['name'].startswith('soak-')];summary['soak']={'requests':len(soak),'sum_request_seconds':sum(r.get('elapsed_s',0) for r in soak),'failures':sum(not r['passed'] for r in soak)} long=[r for r in rows if r['name']=='long-output-16384'];summary['long_output']=[{k:v for k,v in r.items() if k not in ('content','reasoning')} for r in long] quality=a.directory/(variant+'-quality.jsonl') if quality.exists(): q=[json.loads(l) for l in quality.read_text().splitlines()];summary['quality']={'tasks':len(q),'failures':[r['name'] for r in q if not r['passed']]} result[variant]=summary if all(v in result for v in ['baseline','mtp2']): result['speedup']={k:result['mtp2']['timing'][k]['decode_tps_median']/v['decode_tps_median'] for k,v in result['baseline']['timing'].items() if k in result['mtp2']['timing']} for variant in ['baseline','mtp2']: path=a.directory/(variant+'-quality-medium.jsonl') if path.exists() and variant in result: q=[json.loads(l) for l in path.read_text().splitlines()];result[variant]['quality_medium']={'tasks':len(q),'failures':[r['name'] for r in q if not r['passed']]} rpath=a.directory/'resources.jsonl' if rpath.exists(): rows=[json.loads(l) for f in [rpath,a.directory/'resources-resumed.jsonl'] if f.exists() for l in f.read_text().splitlines() if l];gpus=[] for r in rows: try:gpus.append([float(x.strip()) for x in r['gpu'].split(',')]) except (ValueError,KeyError):pass result['resources']={'samples':len(rows),'min_host_available_gib':min(r['available_gib'] for r in rows),'oom_kill_max':max(r['oom_kill'] for r in rows),'safeguard_triggered':any(r['stop'] for r in rows),'gpu_memory_peak_mib':max((g[0] for g in gpus),default=None),'gpu_temperature_peak_c':max((g[3] for g in gpus),default=None)} print(json.dumps(result,ensure_ascii=False,indent=2))