21 lines
1016 B
Python
21 lines
1016 B
Python
import json,statistics,sys
|
|
from pathlib import Path
|
|
result={}
|
|
for fn in sys.argv[1:]:
|
|
rows=[]
|
|
for line in Path(fn).read_text().splitlines():
|
|
try:r=json.loads(line)
|
|
except ValueError:continue
|
|
if r.get('event')=='RESULT':rows.append(r)
|
|
label=rows[0]['label'];groups={}
|
|
for name in ['short','prefill-8192','reuse-8192','prefill-32768','reuse-32768']:
|
|
rr=[r for r in rows if not r['profile'] and r['name'].rsplit('-',1)[0]==name]
|
|
if not rr:continue
|
|
groups[name]={'n':len(rr),'correct':sum(r['correct'] for r in rr)}
|
|
for k in ['ttft_s','elapsed_s','decode_tps_approx']:
|
|
values=[r[k] for r in rr];groups[name][k]={'median':statistics.median(values),'values':values}
|
|
groups[name]['output_tokens']=[r['usage']['completion_tokens'] for r in rr]
|
|
groups[name]['reasoning_tokens']=[r['usage'].get('completion_tokens_details',{}).get('reasoning_tokens') for r in rr]
|
|
result[label]={'total_requests':len(rows),'correct':sum(r['correct'] for r in rows),'groups':groups}
|
|
print(json.dumps(result,indent=2))
|