Record successful 2048-token CUDA Graph retest and preserve failure samples

scripts/benchmark.py

@@ -19,13 +19,18 @@ parser = argparse.ArgumentParser()
 parser.add_argument('--label', required=True)
 parser.add_argument('--long-max-tokens', type=int, default=1024)
 parser.add_argument('--phase', choices=['all', 'short', 'long'], default='all')
+parser.add_argument('--continue-on-answer-failure', action='store_true',
+                    help='Collect all cases; still exit nonzero if any answer fails.')
 args = parser.parse_args()
 BASE = 'http://127.0.0.1:8000'
 KEY = Path('/run/secrets/qwen_api_key').read_text().strip()
 HEADERS = {'Authorization': 'Bearer ' + KEY, 'Content-Type': 'application/json'}
+emit_lock = threading.Lock()
+answer_failures = []
 
 def emit(record):
-    print(json.dumps({'label': args.label, **record}, ensure_ascii=False), flush=True)
+    with emit_lock:
+        print(json.dumps({'label': args.label, **record}, ensure_ascii=False), flush=True)
 
 def metrics():
     request = urllib.request.Request(BASE + '/metrics', headers=HEADERS)
@@ -86,7 +91,10 @@ def run(name, prompt, expected=None, barrier=None, max_tokens=512):
               'correct': bool(content.strip()) and (expected is None or expected in content)}
     emit(record)
     if not record['correct'] or finish != 'stop':
-        raise AssertionError('Failed answer or truncated output: ' + name)
+        if not args.continue_on_answer_failure:
+            raise AssertionError('Failed answer or truncated output: ' + name)
+        answer_failures.append(name)
+        emit({'event': 'ANSWER_CHECK_FAILED', 'test': name})
     return record
 
 emit({'event': 'start', 'phase': args.phase})
@@ -125,4 +133,6 @@ if args.phase in ('all', 'long'):
         after = metrics()
         emit({'test':f'prefix_{size}', 'same_output':a['output_sha256']==b['output_sha256'],
               'metrics_delta':{key:after[key]-before.get(key,0) for key in after}})
-emit({'event':'BENCHMARK_PASS'})
+emit({'event': 'BENCHMARK_FAILED' if answer_failures else 'BENCHMARK_PASS',
+      'answer_failures': answer_failures})
+raise SystemExit(1 if answer_failures else 0)