Qualify and enable MTP 2 with paired workload evidence and fallback

scripts/quality_tasks.py

@@ -0,0 +1,54 @@
+"""Normal-EOS task checks: extraction, scheduling, tool arguments, and guarded code tests."""
+import argparse,ast,json,subprocess,time,urllib.request
+from pathlib import Path
+p=argparse.ArgumentParser();p.add_argument('--output',type=Path,required=True);p.add_argument('--only');p.add_argument('--effort',default='xhigh',choices=['low','medium','xhigh']);p.add_argument('--max-tokens',type=int,default=8192);a=p.parse_args();key=(Path(__file__).resolve().parents[1]/'secrets/api-key').read_text().strip()
+def call(messages,**extra):
+ body=dict(model='qwen3.8-flash-next',messages=messages,temperature=0,seed=6000,max_tokens=a.max_tokens,reasoning_effort=a.effort);body.update(extra)
+ req=urllib.request.Request('http://127.0.0.1:8000/v1/chat/completions',data=json.dumps(body).encode(),headers={'Authorization':'Bearer '+key,'Content-Type':'application/json'})
+ with urllib.request.urlopen(req,timeout=600) as r:return json.load(r)
+# Only independently expected data; JSON parse accepts whitespace, not Markdown wrappers.
+cases=[('invoice-extraction','Extract this invoice to JSON with keys supplier, currency, total, due_date: Supplier North Star Ltd; currency CNY; rows: 3 items at 125 each, 2 items at 48 each; shipping 20; discount 15. Due date 2026-10-03. total must be an integer. Reply only JSON.',{'supplier':'North Star Ltd','currency':'CNY','total':476,'due_date':'2026-10-03'}),('deduplicate','Input orders: A pending 12; B paid 20; A paid 15; C cancelled 99; D paid 8; B paid 25. Keep the last occurrence of each ID, discard cancelled orders, and output ONLY JSON {"ids":[...],"total":...}. Sort IDs alphabetically.',{'ids':['A','B','D'],'total':48}),('schedule','Jobs A takes 3 hours; B takes 5 hours after A; C takes 4 hours after A; D takes 2 hours after both B and C. Unlimited workers, start at hour 0. Output only JSON with earliest_finish and critical_path (array of job letters).',{'earliest_finish':10,'critical_path':['A','B','D']}),('chinese-policy','制度：金额不超过5000元由主管审批；超过5000且不超过20000由经理审批；超过20000由总监审批。加急不改变审批人。申请：甲5000，乙5001加急，丙20000，丁20001。请仅输出JSON，键为甲乙丙丁，值为主管/经理/总监。',{'甲':'主管','乙':'经理','丙':'经理','丁':'总监'})]
+# No imports or arbitrary method calls permitted; child has CPU and address-space limits.
+harness='''import ast,json,resource,sys
+resource.setrlimit(resource.RLIMIT_CPU,(2,2));resource.setrlimit(resource.RLIMIT_AS,(256*1024**2,256*1024**2))
+x=json.load(sys.stdin);tree=ast.parse(x['code'])
+for n in ast.walk(tree):
+ if isinstance(n,(ast.Import,ast.ImportFrom,ast.ClassDef,ast.With,ast.AsyncWith,ast.Global,ast.Nonlocal)):raise ValueError('disallowed syntax')
+ if isinstance(n,ast.Name) and n.id.startswith('__'):raise ValueError('dunder name')
+ if isinstance(n,ast.Attribute) and n.attr not in ('append','sort','copy'):raise ValueError('disallowed attribute')
+ if isinstance(n,ast.Call):
+  if isinstance(n.func,ast.Name) and n.func.id not in ('sorted','len','range','min','max','list','enumerate','sum'):raise ValueError('disallowed call')
+  if not isinstance(n.func,(ast.Name,ast.Attribute)):raise ValueError('indirect call')
+ if isinstance(n,ast.FunctionDef) and (n.decorator_list or n.returns or any(z.annotation for z in n.args.args)):raise ValueError('annotations/decorators unsupported')
+if any(not isinstance(n,ast.FunctionDef) for n in tree.body):raise ValueError('functions only')
+g={'__builtins__':{k:__builtins__.__dict__[k] for k in ['sorted','len','range','min','max','list','enumerate','sum']}}
+exec(compile(tree,'candidate','exec'),g)
+for args,expected in x['tests']:
+ result=g[x['function']](*args)
+ if result!=expected:raise ValueError('wrong result: '+repr(result))
+print('passed')
+'''
+codecases=[('merge-intervals','merge_intervals','Merge overlapping integer intervals, including touching endpoints. Return sorted merged intervals as lists.',[[[[]],[]],[[[[1,3],[2,6],[8,10],[15,18]]],[[1,6],[8,10],[15,18]]],[[[[1,4],[4,5]]],[[1,5]]],[[[[5,7],[1,9],[2,3]]],[[1,9]]]]),('binary-search','lower_bound','Return the index of the first element >= target in a sorted list nums, or len(nums) if none. Parameters nums, target.',[[[[],3],0],[[[1,2,2,4],2],1],[[[1,2,2,4],3],3],[[[1,2,2,4],9],4],[[[1,2,2,4],0],0]])]
+with a.output.open('x') as out:
+ def save(row):out.write(json.dumps(row,ensure_ascii=False)+'\n');out.flush();print(json.dumps({k:v for k,v in row.items() if k!='response'},ensure_ascii=False),flush=True)
+ for name,prompt,expected in cases:
+  if a.only and a.only!=name:continue
+  r=None
+  start=time.monotonic()
+  try:
+   r=call([{'role':'user','content':prompt}]);c=r['choices'][0];actual=json.loads(c['message']['content']);passed=actual==expected and c['finish_reason']=='stop';save(dict(name=name,passed=passed,elapsed_s=time.monotonic()-start,response=r))
+  except Exception as e:save(dict(name=name,passed=False,error=str(e),response=r))
+ for name,fn,task,tests in codecases:
+  if a.only and a.only!=name:continue
+  r=None
+  try:
+   r=call([{'role':'user','content':f'Write Python function {fn}. {task} Reply ONLY a JSON object with a code string. Use no imports, type annotations, decorators, top-level statements or helper functions. Allowed builtins: sorted,len,range,min,max,list,enumerate,sum. Allowed methods: append,sort,copy.'}]);c=r['choices'][0];code=json.loads(c['message']['content'])['code'];result=subprocess.run(['python3','-I','-c',harness],input=json.dumps(dict(code=code,function=fn,tests=tests)),capture_output=True,text=True,timeout=5);save(dict(name=name,passed=result.returncode==0 and c['finish_reason']=='stop',validation_stdout=result.stdout,validation_stderr=result.stderr,response=r))
+  except Exception as e:save(dict(name=name,passed=False,error=str(e),response=r))
+ tools=[dict(type='function',function=dict(name='create_ticket',description='Create a support ticket',parameters=dict(type='object',properties={'title':{'type':'string'},'priority':{'type':'string','enum':['low','normal','urgent']},'asset':{'type':'string'}},required=['title','priority','asset'])))]
+ r=None
+ try:
+  if a.only and a.only!='tool-arguments':
+   print('QUALITY_COMPLETE',flush=True);raise SystemExit(0)
+  r=call([{'role':'user','content':'Call create_ticket with title exactly GPU cooling failure, priority urgent, and asset exactly GPU-092.'}],tools=tools,tool_choice='auto');c=r['choices'][0];tc=c['message']['tool_calls'];passed=len(tc)==1 and tc[0]['function']['name']=='create_ticket' and json.loads(tc[0]['function']['arguments'])=={'title':'GPU cooling failure','priority':'urgent','asset':'GPU-092'};save(dict(name='tool-arguments',passed=passed,response=r))
+ except Exception as e:save(dict(name='tool-arguments',passed=False,error=str(e),response=r))
+print('QUALITY_COMPLETE',flush=True)