Compare MTP steps and prefill batches; retain qualified MTP 2 baseline

scripts/qualification.py

@@ -1,7 +1,7 @@
 """Paired API qualification. Run identically on baseline and MTP; synthetic data only."""
 import argparse,base64,concurrent.futures,hashlib,json,random,time,urllib.request
 from pathlib import Path
-p=argparse.ArgumentParser();p.add_argument('--output',type=Path,required=True);p.add_argument('--soak-seconds',type=int,default=600);p.add_argument('--assets',type=Path,default=Path('/data/flash-next/cache/qualification'));a=p.parse_args()
+p=argparse.ArgumentParser();p.add_argument('--output',type=Path,required=True);p.add_argument('--soak-seconds',type=int,default=600);p.add_argument('--screen',action='store_true',help='Skip 16K output and soak for parameter screening');p.add_argument('--assets',type=Path,default=Path('/data/flash-next/cache/qualification'));a=p.parse_args()
 key=(Path(__file__).resolve().parents[1]/'secrets/api-key').read_text().strip();url='http://127.0.0.1:8000';model='qwen3.8-flash-next'
 def req(path,body=None):
  return urllib.request.urlopen(urllib.request.Request(url+path,data=None if body is None else json.dumps(body).encode(),headers={'Authorization':'Bearer '+key,'Content-Type':'application/json'}),timeout=1200)
@@ -65,7 +65,7 @@ with a.output.open('x') as out:
  for tag,prompt in prompts:
   record(run('warmup-'+tag,text(prompt),256,forced=True,effort='xhigh'))
   for i in range(3):record(run(f'perf-{tag}-{i}',text(prompt),2048,forced=True,effort='xhigh'))
- record(run('long-output-16384',text('Write an extensive technical textbook chapter explaining algorithms, with many worked examples and Python implementations. Continue with additional examples throughout.'),16384,forced=True,effort='xhigh'))
+ if not a.screen:record(run('long-output-16384',text('Write an extensive technical textbook chapter explaining algorithms, with many worked examples and Python implementations. Continue with additional examples throughout.'),16384,forced=True,effort='xhigh'))
  # Exact token count, fresh random filler, two needles at separated positions.
  rng=random.Random(927);lines=[f'Item {i:06d}: batch {rng.randrange(10**8):08d}, archived in shelf {rng.randrange(1000):03d}.\n' for i in range(16000)]
  lines[20]='The northern access code is NORTH-49273.\n';lines[2500]='The southern access code is SOUTH-18362.\n'
@@ -87,7 +87,7 @@ with a.output.open('x') as out:
   fs=[pool.submit(run,f'queued-{i}',text(f'Compute {43+i} * 17. Reply only with the integer.'),2048,str((43+i)*17)) for i in range(4)]
   for f in concurrent.futures.as_completed(fs):record(f.result())
  start=time.monotonic();i=0
- while time.monotonic()-start<a.soak_seconds:
+ while not a.screen and time.monotonic()-start<a.soak_seconds:
   mode=i%5
   if mode==0:r=run(f'soak-{i}-long',longmessages,4096,['NORTH-49273','SOUTH-18362'])
   elif mode==1:r=run(f'soak-{i}-image',image_msgs,4096,['SPARK-82741','1368'])

scripts/summarize_tuning.py

@@ -0,0 +1,20 @@
+"""Summarize tuning receipts without omitting failed variants."""
+import argparse,json,pathlib,re,statistics
+p=argparse.ArgumentParser();p.add_argument('root',type=pathlib.Path);a=p.parse_args();out={}
+for folder in sorted(a.root.glob('*/*')):
+ if not folder.is_dir():continue
+ entry={};status=folder/'status.json'
+ if status.exists():entry.update(json.loads(status.read_text()))
+ for f in ['screen.jsonl','quality-medium.jsonl']:
+  path=folder/f
+  if not path.exists():continue
+  rows=[json.loads(l) for l in path.read_text().splitlines() if l]
+  entry[f]={'count':len(rows),'failed':[r['name'] for r in rows if not r['passed']]}
+  if f=='screen.jsonl':
+   entry['decode_tps']={tag:statistics.median(r['decode_tps'] for r in rows if r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps')) for tag in ['code','chinese','reasoning'] if any(r['name'].startswith('perf-'+tag+'-') and r.get('decode_tps') for r in rows)}
+   entry['latency']={r['name']:{'ttft_s':r.get('ttft_s'),'elapsed_s':r.get('elapsed_s'),'passed':r['passed']} for r in rows if r['name'] in ['long-context-0','long-context-1','long-context-image','video-120s','multi-image']}
+ log=folder/'server.log'
+ if log.exists():
+  s=log.read_text();entry['memory_log']=[l for l in s.splitlines() if any(t in l for t in ['Model loading took','Available KV cache memory','GPU KV cache size','To serve at least one request'])]
+ if entry:out[str(folder.relative_to(a.root))]=entry
+print(json.dumps(out,ensure_ascii=False,indent=2))

scripts/tune_screen.py

@@ -0,0 +1,55 @@
+"""Bounded screening runner. Saves evidence and restores original compose on exit."""
+import argparse,json,os,pathlib,subprocess,time,urllib.request
+p=argparse.ArgumentParser();p.add_argument('--steps',nargs='+',type=int,default=[2,1,3]);p.add_argument('--batch',type=int,default=2048);p.add_argument('--run-id',required=True);a=p.parse_args()
+root=pathlib.Path(__file__).resolve().parents[1];os.chdir(root)
+audit=pathlib.Path('/data/flash-next/audit/tuning')/a.run_id;audit.mkdir(parents=True,exist_ok=False)
+original=(root/'compose.yaml').read_text();(audit/'original.yaml').write_text(original)
+key=(root/'secrets/api-key').read_text().strip()
+def cmd(args,**kwargs):return subprocess.run(args,check=True,**kwargs)
+def state():return json.loads(subprocess.check_output(['docker','inspect','qwen38-flash-6000d']))[0]
+def ready(limit=420):
+ start=time.monotonic()
+ while time.monotonic()-start<limit:
+  try:
+   with urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3) as r:
+    if r.status==200:return True
+  except Exception:pass
+  if state()['State']['Status']=='exited':return False
+  time.sleep(5)
+ return False
+def capture(folder):
+ s=subprocess.check_output(['docker','logs','qwen38-flash-6000d'],stderr=subprocess.STDOUT).decode();(folder/'server.log').write_text(s.replace(key,'<REDACTED>'))
+ try:
+  req=urllib.request.Request('http://127.0.0.1:8000/metrics',headers={'Authorization':'Bearer '+key})
+  with urllib.request.urlopen(req,timeout=5) as r:metrics=r.read().decode()
+  (folder/'mtp-metrics.txt').write_text('\n'.join(l for l in metrics.splitlines() if 'spec_decode' in l))
+ except Exception:pass
+monlog=(audit/'resources.jsonl').open('x');monitor=subprocess.Popen(['python3','scripts/monitor.py'],stdout=monlog,stderr=subprocess.STDOUT)
+(audit/'monitor.pid').write_text(str(monitor.pid))
+try:
+ for n in a.steps:
+  folder=audit/f'mtp{n}-b{a.batch}';folder.mkdir()
+  config=original.replace('num_speculative_tokens":2','num_speculative_tokens":'+str(n)).replace('"cudagraph_capture_sizes":[1,3]','"cudagraph_capture_sizes":[1,'+str(n+1)+']').replace('--max-num-batched-tokens 2048','--max-num-batched-tokens '+str(a.batch)).replace('restart: unless-stopped','restart: "no"')
+  (folder/'compose.yaml').write_text(config)
+  # Run current MTP2 as-is for fresh paired control; all later candidates restart.
+  same=n==2 and a.batch==2048 and a.steps.index(n)==0
+  if not same:
+   (root/'compose.yaml').write_text(config);cmd(['docker','compose','--progress','plain','up','-d','--force-recreate'],timeout=120)
+  if not ready():
+   capture(folder);(folder/'status.json').write_text(json.dumps({'startup':False}));print('STARTUP_FAILED',folder,flush=True);continue
+  for script,name,extra in [('qualification.py','screen.jsonl',['--screen']),('quality_tasks.py','quality-medium.jsonl',['--effort','medium'])]:
+   with (folder/(name+'.log')).open('x') as log:
+    result=subprocess.run(['python3','scripts/'+script,'--output',str(folder/name)]+extra,stdout=log,stderr=subprocess.STDOUT,timeout=1200)
+   if result.returncode!=0:print('SCRIPT_FAILED',folder,script,result.returncode,flush=True)
+  capture(folder);rows=[]
+  for name in ['screen.jsonl','quality-medium.jsonl']:
+   f=folder/name
+   if f.exists():rows += [json.loads(l) for l in f.read_text().splitlines()]
+  status={'startup':True,'records':len(rows),'failures':[r['name'] for r in rows if not r['passed']]}
+  (folder/'status.json').write_text(json.dumps(status,indent=2));print('SCREEN_COMPLETE',folder,status,flush=True)
+finally:
+ (root/'compose.yaml').write_text(original)
+ cmd(['docker','compose','--progress','plain','up','-d','--force-recreate'],timeout=120)
+ restored=ready();(audit/'restored.json').write_text(json.dumps({'healthy':restored,'time':time.time()}))
+ monitor.terminate();monitor.wait(timeout=15);monlog.close()
+ print('RESTORED',restored,flush=True)