45 lines
2.6 KiB
Python
45 lines
2.6 KiB
Python
"""Summarize context-benchmark records, keeping first attempts separate from retries."""
|
|
import argparse
|
|
import json
|
|
import statistics
|
|
from pathlib import Path
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('files', nargs='+', type=Path)
|
|
args = parser.parse_args()
|
|
for path in args.files:
|
|
rows = []
|
|
for line in path.read_text().splitlines():
|
|
if line.startswith('{'):
|
|
rows.append(json.loads(line))
|
|
summary = {'file': path.name,
|
|
'completed': any(r.get('event') == 'CONTEXT_SUITE_COMPLETED' for r in rows),
|
|
'note': 'Request completion is not a host-stability verdict; inspect resource and kernel OOM evidence.',
|
|
'stages': []}
|
|
for size in [65536, 131072, 260000]:
|
|
results = [r for r in rows if r.get('event') == 'RESULT' and r['target_input_tokens'] == size]
|
|
first = [r for r in results if r['attempt'] == 1]
|
|
if not first:
|
|
if any(r.get('event') == 'STAGE_START' and r['target_input_tokens'] == size for r in rows):
|
|
summary['stages'].append({'target_input_tokens': size, 'first_attempts_completed': 0,
|
|
'status': 'started_without_completed_response'})
|
|
continue
|
|
cold = [r for r in first if r['case'] == 'front']
|
|
warm = [r for r in first if r['case'] != 'front']
|
|
summary['stages'].append({
|
|
'target_input_tokens': size, 'first_attempts': len(first),
|
|
'correct_first_attempts': sum(r['correct'] for r in first),
|
|
'truncated_first_attempts': sum(r['finish_reason'] == 'length' for r in first),
|
|
'retries': len(results)-len(first),
|
|
'actual_input_range': [min(r['usage']['prompt_tokens'] for r in first), max(r['usage']['prompt_tokens'] for r in first)],
|
|
'first_document_ttft_s': [r['ttft_s'] for r in cold],
|
|
'reused_prefix_ttft_median_s': statistics.median(r['ttft_s'] for r in warm) if warm else None,
|
|
'first_document_elapsed_s': [r['elapsed_s'] for r in cold],
|
|
'reused_prefix_elapsed_median_s': statistics.median(r['elapsed_s'] for r in warm) if warm else None,
|
|
'max_completion_tokens': max(r['usage']['completion_tokens'] for r in first),
|
|
'max_reasoning_tokens': max(r['usage'].get('completion_tokens_details', {}).get('reasoning_tokens', 0) for r in first),
|
|
'prefix_hit_tokens': sum(r['metrics_delta']['vllm:prefix_cache_hits_total'] for r in results),
|
|
'preemptions': sum(r['metrics_delta']['vllm:num_preemptions_total'] for r in results),
|
|
})
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|