ruihan/qwen38-flash-next-dgx-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record successful 2048-token CUDA Graph retest and preserve failure samples

Browse Source
This commit is contained in:
ruihan
2026-09-17 23:04:40 +08:00
parent 1e2f48d1a4
commit 48a1c9d7c4
19 changed files with 430 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark.py
+13 -3
View File
@@ -19,13 +19,18 @@ parser = argparse.ArgumentParser()
parser.add_argument('--label', required=True)
parser.add_argument('--long-max-tokens', type=int, default=1024)
parser.add_argument('--phase', choices=['all', 'short', 'long'], default='all')
parser.add_argument('--continue-on-answer-failure', action='store_true',
help='Collect all cases; still exit nonzero if any answer fails.')
args = parser.parse_args()
BASE = 'http://127.0.0.1:8000'
KEY = Path('/run/secrets/qwen_api_key').read_text().strip()
HEADERS = {'Authorization': 'Bearer ' + KEY, 'Content-Type': 'application/json'}
emit_lock = threading.Lock()
answer_failures = []
def emit(record):
print(json.dumps({'label': args.label, **record}, ensure_ascii=False), flush=True)
with emit_lock:
print(json.dumps({'label': args.label, **record}, ensure_ascii=False), flush=True)
def metrics():
request = urllib.request.Request(BASE + '/metrics', headers=HEADERS)
@@ -86,7 +91,10 @@ def run(name, prompt, expected=None, barrier=None, max_tokens=512):
'correct': bool(content.strip()) and (expected is None or expected in content)}
emit(record)
if not record['correct'] or finish != 'stop':
raise AssertionError('Failed answer or truncated output: ' + name)
if not args.continue_on_answer_failure:
raise AssertionError('Failed answer or truncated output: ' + name)
answer_failures.append(name)
emit({'event': 'ANSWER_CHECK_FAILED', 'test': name})
return record
emit({'event': 'start', 'phase': args.phase})
@@ -125,4 +133,6 @@ if args.phase in ('all', 'long'):
after = metrics()
emit({'test':f'prefix_{size}', 'same_output':a['output_sha256']==b['output_sha256'],
'metrics_delta':{key:after[key]-before.get(key,0) for key in after}})
emit({'event':'BENCHMARK_PASS'})
emit({'event': 'BENCHMARK_FAILED' if answer_failures else 'BENCHMARK_PASS',
'answer_failures': answer_failures})
raise SystemExit(1 if answer_failures else 0)
scripts/summarize-benchmark.py
+4 -4
View File
@@ -14,13 +14,13 @@ for path in args.files:
rows.append(json.loads(line))
except ValueError:
continue
short = [r for r in rows if r.get('test', '').startswith('short_')]
short = [r for r in rows if r.get('test', '').startswith('short_') and 'correct' in r]
summary = {'file': path.name,
'completed': any(r.get('event') == 'BENCHMARK_PASS' for r in rows),
'completed': any(r.get('event') in ('BENCHMARK_PASS', 'BENCHMARK_FAILED', 'BENCHMARK_COMPLETED') for r in rows),
'short_decode_tps_median': statistics.median(r['decode_tps_approx'] for r in short) if short else None,
'short_ttft_s_median': statistics.median(r['ttft_s'] for r in short) if short else None,
'all_checked_answers_correct': all(r['correct'] for r in rows if 'correct' in r),
'long': [{k:r[k] for k in ['test','ttft_s','elapsed_s','correct']} for r in rows if r.get('test','').startswith('long_')],
'all_checked_answers_correct': bool([r for r in rows if 'correct' in r]) and all(r['correct'] and r.get('finish_reason') == 'stop' for r in rows if 'correct' in r),
'long': [{k:r[k] for k in ['test','ttft_s','elapsed_s','correct']} for r in rows if r.get('test','').startswith('long_') and 'correct' in r],
'concurrency': [r for r in rows if r.get('test') == 'concurrency_summary'],
'prefix': [r for r in rows if r.get('test','').startswith('prefix_')]}
print(json.dumps(summary, ensure_ascii=False, indent=2))