docs(perf): profile CUDA graph coverage and PLE costs on Spark

experiments/profile128k/README.md

@@ -0,0 +1,38 @@
+# Nsight Systems 诊断脚本
+
+固定 nightly `0bfc7a15`、当前 PLE 适配，单 GPU、串行请求。
+方案见 [plan.md](plan.md)。这些脚本是实验工具，不是默认启动入口。
+
+- `prepare.py prepare --data ...`：使用正在运行的同一模型生成 8K/32K 固定检索数据并预检 token 数。
+- `instrument.py`：读取同目录 `vllm_ple_mmap.original.py` 和 `cudagraph_utils.original.py`，
+  精确匹配源码后生成临时 NVTX 版本；保持算子签名与计算不变。原始文件来自容器，不提交重复副本。
+- `client.py --label eager|graph`：在测试容器内运行，数据挂到 `/profiles/data`，密钥读挂载文件。
+  三轮正常计时和三次独立 trace；所有请求相同，temperature=0、seed=42、reasoning_effort=low。
+- `monitor.py`：在 Spark 宿主机运行，写资源采样到 stdout；以脚本目录中的文件发出停止信号。
+  必须在启动实验前启动监控，并配套退出后恢复原配置的 runner。保护是尽力而为，不能保证拦截瞬时 OOM。
+- `summarize_trace.py <trace.sqlite> ...`：导出 CUDA/NVTX 汇总，不读取进程命令行和环境。
+
+Nsight 使用主机安装目录只读挂载到 `/opt/nsight`。临时服务添加：
+
+```text
+--max-model-len 131072 --profiler-config '{"profiler":"cuda"}'
+```
+
+在原有 `vllm serve ...` 前包裹：
+
+```text
+/opt/nsight/bin/nsys profile --sample=none --trace=cuda,nvtx,osrt \
+  --capture-range=cudaProfilerApi --capture-range-end=repeat:3 --kill=none \
+  --trace-fork-before-exec=true --cuda-graph-trace=node --force-overwrite=true \
+  -o /profiles/<label>/trace vllm serve ...
+```
+
+临时将服务绑定到容器回环地址，客户端使用 `docker exec`；保留已有缓存、代理和 secret 挂载。
+PLE 和 CudaGraphManager 的标记文件分别只读覆盖原模块，不修改镜像或权重。
+测试时自动重启关闭；原 Compose 保留备份，退出时恢复并做健康及 smoke 检查。
+
+计时阶段没有启动 capture，但 Nsight launcher 和 NVTX 包装仍存在；本轮不是完全卸载 profiler 的独立测速。
+node 级采样可能增加开销，因此 trace 时长只用作诊断，不能代替无采样阶段排名。
+CPU API、NVTX 和 GPU 时间存在重叠；GPU busy 是 kernel/copy/memset 区间并集，
+分母为首次至末次 GPU 活动的时间跨度，不是 SM 利用率、带宽利用率或完整 HTTP 延迟。
+原始 `.nsys-rep` / SQLite 可能包含进程参数，只保留在 Spark 的受限目录，不提交 Git。

experiments/profile128k/client.py

@@ -0,0 +1,67 @@
+import argparse,json,hashlib,time,urllib.request,signal
+from pathlib import Path
+parser=argparse.ArgumentParser();parser.add_argument('--label',required=True);args=parser.parse_args()
+base='http://127.0.0.1:8000'
+key=Path('/run/secrets/qwen_api_key').read_text().strip()
+headers={'Authorization':'Bearer '+key,'Content-Type':'application/json'}
+def emit(r):print(json.dumps({'label':args.label,'time':time.time(),**r},ensure_ascii=False),flush=True)
+def request(path,payload=None):
+ req=urllib.request.Request(base+path,headers=headers,data=json.dumps(payload).encode() if payload is not None else None)
+ with urllib.request.urlopen(req,timeout=60) as res:return res.read()
+def metrics():
+ result={}
+ for l in request('/metrics').decode().splitlines():
+  if l.startswith('vllm:ple_mmap_') or l.startswith('vllm:prefix_cache_'):
+   name=l.split('{')[0].split()[0]
+   if name.endswith('_total'):result[name]=result.get(name,0)+float(l.split()[-1])
+ return result
+def deadline(*_):raise TimeoutError('Request exceeded 180 seconds')
+signal.signal(signal.SIGALRM,deadline)
+def run(name,prompt,expected=None,profile=False):
+ m=[{'role':'user','content':prompt}]
+ count=json.loads(request('/tokenize',{'model':'qwen3.8-flash-next','messages':m,'chat_template_kwargs':{'reasoning_effort':'low'}}))['count']
+ assert count+2048<=131072
+ before=metrics();payload={'model':'qwen3.8-flash-next','messages':m,'temperature':0,'seed':42,'reasoning_effort':'low','max_tokens':2048 if expected else 512,'stream':True,'stream_options':{'include_usage':True}}
+ req=urllib.request.Request(base+'/v1/chat/completions',headers=headers,data=json.dumps(payload,ensure_ascii=False).encode())
+ emit({'event':'REQUEST_START','name':name,'profile':profile,'prompt_tokens_preflight':count})
+ if profile:request('/start_profile',{})
+ t=time.monotonic();first=last=visible=None;content=reasoning='';usage={};finish=None
+ signal.alarm(180)
+ try:
+  with urllib.request.urlopen(req,timeout=180) as res:
+   for raw in res:
+    line=raw.decode().strip()
+    if not line.startswith('data: ') or line=='data: [DONE]':continue
+    x=json.loads(line[6:])
+    if x.get('error'):raise RuntimeError('Streaming error')
+    if x.get('usage'):usage=x['usage']
+    for c in x.get('choices',[]):
+     d=c.get('delta',{});a=d.get('content') or '';r=d.get('reasoning') or d.get('reasoning_content') or ''
+     if a or r:
+      now=time.monotonic();first=first if first is not None else now;last=now
+     if a:visible=visible if visible is not None else now
+     content+=a;reasoning+=r;finish=c.get('finish_reason') or finish
+ finally:
+  signal.alarm(0)
+  if profile:request('/stop_profile',{})
+ elapsed=time.monotonic()-t
+ after=metrics()
+ correct=bool(content.strip()) and (expected is None or content.strip()==expected) and finish=='stop'
+ emit({'event':'RESULT','name':name,'profile':profile,'correct':correct,'content':content,'finish_reason':finish,'usage':usage,'max_tokens':payload['max_tokens'],'prompt_sha256':hashlib.sha256(prompt.encode()).hexdigest(),'output_sha256':hashlib.sha256((reasoning+'\0'+content).encode()).hexdigest(),'ttft_s':first-t if first else None,'first_content_s':visible-t if visible else None,'elapsed_s':elapsed,'decode_tps_approx':(usage.get('completion_tokens',0)-1)/(last-first) if last and first and last>first else None,'metrics_delta':{k:after[k]-before.get(k,0) for k in after}})
+ assert correct and usage['prompt_tokens']==count,name
+ return prompt
+short='用中文写一段约150字的说明，解释数据库索引为什么能加快查询，以及它对写入有什么影响。'
+run('warmup','计算17乘19，只输出结果。','323')
+# Same prefix fixtures across configurations; phase/round identity is before the document.
+fixtures={n:json.loads(Path(f'/profiles/data/ctx-{n}-1.json').read_text())['cases'][1] for n in [8192,32768]}
+for i in range(3):
+ run(f'short-{i}',short)
+ for n,c in fixtures.items():
+  prompt=f'测试文档批次 BENCH-{i}-{n}。\n'+c['messages'][0]['content']
+  run(f'prefill-{n}-{i}',prompt,c['expected'])
+  run(f'reuse-{n}-{i}',prompt,c['expected'])
+run('trace-short',short,profile=True)
+c=fixtures[32768];prompt='测试文档批次 TRACE-32768。\n'+c['messages'][0]['content']
+run('trace-prefill-32768',prompt,c['expected'],profile=True)
+run('trace-reuse-32768',prompt,c['expected'],profile=True)
+emit({'event':'PROFILE_SUITE_PASS'})

experiments/profile128k/instrument.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+import ast
+p=Path(__file__).resolve().parent
+s=(p/'vllm_ple_mmap.original.py').read_text()
+replacements=[
+('            torch.cuda.current_stream(ids.device).synchronize()', '            with torch.cuda.nvtx.range("PLE_wait_for_GPU"):\n                torch.cuda.current_stream(ids.device).synchronize()'),
+('        rows = table.gather(uniq)  # uint8 [U, row_bytes], fresh & writable','        with torch.cuda.nvtx.range("PLE_CPU_gather"):\n            rows = table.gather(uniq)  # uint8 [U, row_bytes], fresh & writable'),
+('        uniq, inverse = np.unique(ids_np, return_inverse=True)', '        with torch.cuda.nvtx.range("PLE_CPU_dedup"):\n            uniq, inverse = np.unique(ids_np, return_inverse=True)')]
+for before,after in replacements:
+ assert s.count(before)==1,(before,s.count(before))
+ s=s.replace(before,after)
+s+='''\n# Temporary diagnostic annotations; no tensor or arithmetic changes.
+_profile_original_lookup_ids = _lookup_ids_impl
+def _lookup_ids_impl(ngram_ids: torch.Tensor, output: torch.Tensor, layer_name: str) -> None:
+    with torch.cuda.nvtx.range("PLE_lookup_tokens=" + str(ngram_ids.shape[0])):
+        return _profile_original_lookup_ids(ngram_ids, output, layer_name)
+'''
+ast.parse(s);(p/'vllm_ple_mmap.profile.py').write_text(s)
+s=(p/'cudagraph_utils.original.py').read_text()
+s+='''\n# Diagnostic dispatch marker; does not change graph selection.
+_profile_original_dispatch = CudaGraphManager.dispatch
+def _profile_dispatch(self, *args, **kwargs):
+    desc = _profile_original_dispatch(self, *args, **kwargs)
+    actual = args[1] if len(args) > 1 else kwargs.get("num_tokens", -1)
+    torch.cuda.nvtx.mark(f"CG_DISPATCH:actual={actual}:mode={desc.cg_mode.name}:padded={desc.num_tokens}")
+    return desc
+CudaGraphManager.dispatch = _profile_dispatch
+'''
+ast.parse(s);(p/'cudagraph_utils.profile.py').write_text(s)

experiments/profile128k/monitor.py

@@ -0,0 +1,41 @@
+import json,time,subprocess,os
+from pathlib import Path
+out=Path(__file__).resolve().parent
+low=[]
+critical=[]
+initial_oom=None
+while not (out/'monitor.stop').exists():
+ mem={l.split(':')[0]:int(l.split()[1]) for l in Path('/proc/meminfo').read_text().splitlines()}
+ vm=dict(l.split() for l in Path('/proc/vmstat').read_text().splitlines())
+ state=subprocess.run(['docker','inspect','--format','{{json .State}}','qwen38-flash-vllm'],capture_output=True,text=True)
+ status=json.loads(state.stdout) if state.returncode==0 else {}
+ if initial_oom is None: initial_oom=int(vm.get('oom_kill',0))
+ swapped=(int(vm['pswpin'])+int(vm['pswpout']))*os.sysconf('SC_PAGE_SIZE')
+ rec={'time':time.time(),'available_kib':mem['MemAvailable'],'swap_used_kib':mem['SwapTotal']-mem['SwapFree'],
+      'pswpin':int(vm['pswpin']),'pswpout':int(vm['pswpout']),'status':status.get('Status'),
+      'oom_killed':status.get('OOMKilled'),'host_oom_kills':int(vm.get('oom_kill',0)),'memory_pressure':Path('/proc/pressure/memory').read_text().strip()}
+ print(json.dumps(rec),flush=True)
+ if mem['MemAvailable']<1024*1024:
+  (out/'GUARD_STOP').write_text('MemAvailable below 1 GiB')
+  subprocess.run(['docker','stop','-t','2','qwen38-flash-vllm'])
+  break
+ if mem['MemAvailable']<2*1024*1024:
+  critical.append(time.monotonic())
+ else:
+  critical=[]
+ if int(vm.get('oom_kill',0))>initial_oom or (len(critical)>=2 and critical[-1]-critical[0]>=5):
+  (out/'GUARD_STOP').write_text('Host OOM counter increased or MemAvailable below 2 GiB for 5s')
+  subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm'])
+  break
+ if mem['MemAvailable']<4*1024*1024:
+  low.append((time.monotonic(),swapped))
+ else:
+  low=[]
+ if len(low)>=7 and low[-1][0]-low[0][0]>=30 and low[-1][1]-low[0][1]>512*1024**2:
+  (out/'GUARD_STOP').write_text('MemAvailable below 4 GiB for 30s with >512 MiB swapping')
+  subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm'])
+  break
+ if status.get('OOMKilled'):
+  (out/'GUARD_STOP').write_text('Docker reported OOMKilled')
+  break
+ time.sleep(1)

experiments/profile128k/plan.md

@@ -0,0 +1,17 @@
+# CUDA Graph 性能剖析方案（2026-09-18）
+
+目的：定位当前 CUDA Graph 约 4.4% 收益的限制，确认捕获范围与 PLE 图外路径的影响。
+先采证据，再决定是否扩大图覆盖；不将推断写成瓶颈结论。
+
+1. 小型 CUDA 程序验证主机 Nsight Systems 在当前镜像中可采集 CUDA 时间线。
+2. 临时服务上下文上限 131072，串行输入不超过 128K；原镜像、模型、MTP=2、BF16 KV、内存比例 0.80 不变。
+3. 对照 eager、小尺寸图 [1,2,4,8,12]。若证据支持，再试含 2048-token 预填充尺寸的图配置。
+4. 固定合成短问答、8K/32K 首次/重复输入；记录延迟、输出/思考 tokens、PLE 分项计数、图重放和内存。
+5. Nsight 使用 API 控制的短时间窗口，只采选定请求；计时基准在采样关闭时运行，trace 不作性能定量排名。
+6. 用 CPU CUDA API、GPU 活动、同步、PLE 指标判断瓶颈；这些时间可能重叠，不简单相加当百分比。
+7. 主机 OOM 计数增加、MemAvailable 单次低于 1 GiB 或持续低于 2 GiB 即停止。
+   保留并检查内核 OOM 日志；测试后恢复原服务和 smoke test。128K 仅在候选通过后做资源/正确性验证。
+8. 原始大型 trace 留在 Spark，仓库保存方案、配置、脚本、摘要及证据索引，不提交凭据。
+
+基础事实：当前图模式为 breakable PIECEWISE，PLE 哈希+CPU mmap 查表在 eager break 中运行。
+大于已捕获 token 数的 batch 无匹配图时回退无图执行。扩大图尺寸仍需实测兼容性与额外内存。

experiments/profile128k/prepare.py

@@ -0,0 +1,198 @@
+"""Generate reproducible long-context retrieval fixtures and test a running vLLM.
+
+Run inside the model container. Never prints the mounted API key.
+"""
+import argparse
+import hashlib
+import json
+import random
+import signal
+import time
+import urllib.request
+from pathlib import Path
+
+SNAPSHOT = '/root/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47'
+MODEL = 'qwen3.8-flash-next'
+MAX_OUTPUT = 2048
+MAX_CONTEXT = 131072
+BASE = 'http://127.0.0.1:8000'
+
+
+def emit(record):
+    print(json.dumps(record, ensure_ascii=False), flush=True)
+
+
+def digest(value):
+    return hashlib.sha256(value.encode()).hexdigest()
+
+
+def headers():
+    key = Path('/run/secrets/qwen_api_key').read_text().strip()
+    return {'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'}
+
+
+def request(path, payload=None, timeout=120):
+    req = urllib.request.Request(BASE + path, headers=headers(),
+          data=json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None)
+    with urllib.request.urlopen(req, timeout=timeout) as response:
+        return json.load(response)
+
+
+def metrics():
+    req = urllib.request.Request(BASE + '/metrics', headers=headers())
+    with urllib.request.urlopen(req, timeout=15) as response:
+        lines = response.read().decode().splitlines()
+    names = ['vllm:prefix_cache_hits_total', 'vllm:prefix_cache_queries_total',
+             'vllm:num_preemptions_total']
+    return {name: sum(float(l.split()[-1]) for l in lines if l.split('{')[0].split(' ')[0] == name)
+            for name in names}
+
+
+def messages(document, project):
+    prompt = ('以下是一份虚构项目预算档案，请按项目编号检索其中的数据。\n' + document +
+              '\n档案结束。\n问题：项目编号 ' + project + ' 的核定预算是多少元？只输出数字。')
+    return [{'role': 'user', 'content': prompt}]
+
+
+def prepare(folder):
+    from transformers import AutoTokenizer
+    tok = AutoTokenizer.from_pretrained(SNAPSHOT, local_files_only=True)
+    folder.mkdir(parents=True, exist_ok=True)
+    for target in [8192, 32768]:
+        for sample in [1, 2]:
+            name = f'ctx-{target}-{sample}'
+            rng = random.Random(target * 10 + sample)
+            amounts = [str(rng.randrange(10000, 99999)) for _ in range(3)]
+            projects = [f'TARGET-{position}-{sample}' for position in ['FRONT', 'MIDDLE', 'END']]
+            filler = [f'项目编号 BG-{i:06d}；核定预算 {rng.randrange(10000,99999)} 元；类别 C{i%19:02d}。' for i in range(20000)]
+
+            def build(n):
+                rows = filler[:n].copy()
+                positions = [int(n * f) for f in [0.05, 0.5, 0.95]]
+                for pos, project, amount in zip(positions, projects, amounts):
+                    rows[pos] = f'项目编号 {project}；核定预算 {amount} 元；类别 C20。'
+                return '文档编号：PROFILE128K-' + name + '\n' + '\n'.join(rows), positions, rows
+
+            low, high = 20, len(filler)
+            while low < high:
+                n = (low + high + 1) // 2
+                doc, _, _ = build(n)
+                count = len(tok.apply_chat_template(messages(doc, projects[1]), tokenize=True, add_generation_prompt=True, return_dict=False))
+                if count <= target:
+                    low = n
+                else:
+                    high = n - 1
+            doc, positions, rows = build(low)
+            cases = []
+            for label, index in [('front', 0), ('middle', 1), ('end', 2), ('front_repeat', 0)]:
+                msgs = messages(doc, projects[index])
+                count = request('/tokenize', {'model': MODEL, 'messages': msgs, 'add_generation_prompt': True, 'chat_template_kwargs': {'reasoning_effort': 'low'}})['count']
+                assert target - 128 <= count <= target + 32, (name, count, target)
+                assert count + MAX_OUTPUT <= MAX_CONTEXT, (name, count)
+                cases.append({'case': label, 'messages': msgs, 'expected': amounts[index],
+                              'prompt_tokens_preflight': count, 'prompt_sha256': digest(msgs[0]['content']),
+                              'max_tokens': MAX_OUTPUT})
+            fractions = [round(len(tok.encode('文档编号：PROFILE128K-'+name+'\n'+'\n'.join(rows[:pos]), add_special_tokens=False)) / cases[0]['prompt_tokens_preflight'], 4)
+                         for pos in positions]
+            fixture = {'document': name, 'target_input_tokens': target, 'sample': sample,
+                       'document_sha256': digest(doc), 'needle_token_fractions': fractions, 'cases': cases}
+            (folder / (name + '.json')).write_text(json.dumps(fixture, ensure_ascii=False))
+            emit({'event': 'FIXTURE_READY', **{k:v for k,v in fixture.items() if k != 'cases'},
+                  'prompt_tokens': [c['prompt_tokens_preflight'] for c in cases]})
+
+
+def deadline(signum, frame):
+    raise TimeoutError('Request exceeded the 1200 second wall-clock limit')
+
+
+def run_case(label, fixture, case, attempt):
+    before = metrics()
+    payload = {'model': MODEL, 'messages': case['messages'], 'temperature': 0, 'seed': 42,
+               'reasoning_effort': 'low', 'max_tokens': MAX_OUTPUT,
+               'stream': True, 'stream_options': {'include_usage': True}}
+    req = urllib.request.Request(BASE + '/v1/chat/completions', headers=headers(),
+                                 data=json.dumps(payload, ensure_ascii=False).encode())
+    start = time.monotonic()
+    first = first_content = last = None
+    content = reasoning = ''
+    usage = {}
+    finish = None
+    signal.alarm(1200)
+    try:
+        with urllib.request.urlopen(req, timeout=1200) as response:
+            for raw in response:
+                line = raw.decode().strip()
+                if not line.startswith('data: ') or line == 'data: [DONE]':
+                    continue
+                item = json.loads(line[6:])
+                if item.get('error'):
+                    raise RuntimeError('Server returned a streaming error')
+                if item.get('usage'):
+                    usage = item['usage']
+                for choice in item.get('choices', []):
+                    delta = choice.get('delta', {})
+                    answer = delta.get('content') or ''
+                    thought = delta.get('reasoning') or delta.get('reasoning_content') or ''
+                    if answer or thought:
+                        now = time.monotonic()
+                        first = first if first is not None else now
+                        last = now
+                    if answer:
+                        first_content = first_content if first_content is not None else now
+                    content += answer
+                    reasoning += thought
+                    finish = choice.get('finish_reason') or finish
+    finally:
+        signal.alarm(0)
+    if finish is None or not usage:
+        raise RuntimeError('Incomplete response or missing usage')
+    after = metrics()
+    elapsed = time.monotonic() - start
+    record = {'event': 'RESULT', 'label': label, 'document': fixture['document'],
+              'target_input_tokens': fixture['target_input_tokens'], 'sample': fixture['sample'],
+              'case': case['case'], 'attempt': attempt, 'expected': case['expected'],
+              'content': content, 'correct': content.strip() == case['expected'] and finish == 'stop',
+              'finish_reason': finish, 'usage': usage, 'max_tokens': MAX_OUTPUT,
+              'prompt_tokens_preflight': case['prompt_tokens_preflight'], 'prompt_sha256': case['prompt_sha256'],
+              'output_sha256': digest(reasoning + '\0' + content),
+              'ttft_s': round(first-start, 4) if first else None,
+              'first_content_s': round(first_content-start, 4) if first_content else None,
+              'elapsed_s': round(elapsed, 4),
+              'decode_tps_approx': round((usage['completion_tokens']-1)/(last-first), 3) if last and first and last>first else None,
+              'metrics_delta': {k: after[k]-before[k] for k in after}}
+    emit(record)
+    assert usage['prompt_tokens'] + MAX_OUTPUT <= MAX_CONTEXT
+    assert usage['prompt_tokens'] == case['prompt_tokens_preflight'], 'Server tokenization differs from preflight'
+    return record
+
+
+def run(folder, label):
+    signal.signal(signal.SIGALRM, deadline)
+    failures = []
+    for target in [8192, 32768]:
+        emit({'event': 'STAGE_START', 'label': label, 'target_input_tokens': target})
+        for sample in [1, 2]:
+            fixture = json.loads((folder / f'ctx-{target}-{sample}.json').read_text())
+            emit({'event': 'DOCUMENT_START', 'label': label, 'document': fixture['document'],
+                  'document_sha256': fixture['document_sha256'], 'needle_token_fractions': fixture['needle_token_fractions']})
+            for case in fixture['cases']:
+                emit({'event': 'REQUEST_START', 'label': label, 'document': fixture['document'], 'case': case['case']})
+                result = run_case(label, fixture, case, 1)
+                if not result['correct']:
+                    failures.append([fixture['document'], case['case']])
+                    run_case(label, fixture, case, 2)
+        emit({'event': 'STAGE_COMPLETED', 'label': label, 'target_input_tokens': target})
+    emit({'event': 'CONTEXT_SUITE_COMPLETED', 'label': label, 'first_attempt_failures': failures})
+    # Quality failures are reported above, not hidden; infrastructure failures raise.
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mode', choices=['prepare', 'run'])
+    parser.add_argument('--data', type=Path, default=Path('/tmp/context262k-data'))
+    parser.add_argument('--label', default='current')
+    args = parser.parse_args()
+    if args.mode == 'prepare':
+        prepare(args.data)
+    else:
+        run(args.data, args.label)

experiments/profile128k/summarize_bench.py

@@ -0,0 +1,20 @@
+import json,statistics,sys
+from pathlib import Path
+result={}
+for fn in sys.argv[1:]:
+ rows=[]
+ for line in Path(fn).read_text().splitlines():
+  try:r=json.loads(line)
+  except ValueError:continue
+  if r.get('event')=='RESULT':rows.append(r)
+ label=rows[0]['label'];groups={}
+ for name in ['short','prefill-8192','reuse-8192','prefill-32768','reuse-32768']:
+  rr=[r for r in rows if not r['profile'] and r['name'].rsplit('-',1)[0]==name]
+  if not rr:continue
+  groups[name]={'n':len(rr),'correct':sum(r['correct'] for r in rr)}
+  for k in ['ttft_s','elapsed_s','decode_tps_approx']:
+   values=[r[k] for r in rr];groups[name][k]={'median':statistics.median(values),'values':values}
+  groups[name]['output_tokens']=[r['usage']['completion_tokens'] for r in rr]
+  groups[name]['reasoning_tokens']=[r['usage'].get('completion_tokens_details',{}).get('reasoning_tokens') for r in rr]
+ result[label]={'total_requests':len(rows),'correct':sum(r['correct'] for r in rows),'groups':groups}
+print(json.dumps(result,indent=2))

experiments/profile128k/summarize_trace.py

@@ -0,0 +1,52 @@
+"""Export only CUDA/NVTX aggregates; omit process arguments and environment."""
+import sqlite3,json,sys,collections
+from pathlib import Path
+
+def merge(intervals):
+ out=[]
+ for a,b in sorted(intervals):
+  if b<=a:continue
+  if out and a<=out[-1][1]:out[-1]=(out[-1][0],max(out[-1][1],b))
+  else:out.append((a,b))
+ return out
+
+def duration(xs):return sum(b-a for a,b in xs)/1e9
+
+def overlap(a,b):
+ i=j=0;total=0
+ while i<len(a) and j<len(b):
+  total+=max(0,min(a[i][1],b[j][1])-max(a[i][0],b[j][0]))
+  if a[i][1]<b[j][1]:i+=1
+  else:j+=1
+ return total/1e9
+
+for fn in sys.argv[1:]:
+ db=sqlite3.connect(fn);db.row_factory=sqlite3.Row
+ tables={r[0] for r in db.execute("select name from sqlite_master where type='table'")}
+ strings=dict(db.execute('select id,value from StringIds'))
+ gpu=[];kerns=[];api=collections.defaultdict(lambda:[0,0]);nv=collections.defaultdict(list);dispatch=collections.Counter();dispatch_timeline=[]
+ for table in ['CUPTI_ACTIVITY_KIND_KERNEL','CUPTI_ACTIVITY_KIND_MEMCPY','CUPTI_ACTIVITY_KIND_MEMSET']:
+  if table in tables:
+   rows=list(db.execute('select * from '+table));gpu.extend((r['start'],r['end']) for r in rows)
+   if table.endswith('KERNEL'):kerns=rows
+ for table in ['CUPTI_ACTIVITY_KIND_RUNTIME','CUPTI_ACTIVITY_KIND_DRIVER']:
+  if table in tables:
+   for r in db.execute('select start,end,nameId from '+table):
+    name=strings.get(r['nameId'],str(r['nameId']));api[name][0]+=1;api[name][1]+=(r['end']-r['start'])/1e9
+ if 'NVTX_EVENTS' in tables:
+  for r in db.execute('select * from NVTX_EVENTS'):
+   keys=r.keys();name=r['text'] if 'text' in keys else None
+   if not name and 'textId' in keys:name=strings.get(r['textId'],'')
+   name=name or ''
+   if name.startswith('CG_DISPATCH:'):
+    dispatch[name]+=1;dispatch_timeline.append({'start_ns':r['start'],'name':name})
+   if name.startswith('PLE_') and r['end'] is not None:nv[name].append((r['start'],r['end']))
+ merged=merge(gpu);span=(merged[-1][1]-merged[0][0])/1e9 if merged else 0
+ nvout={}
+ for name,ranges in nv.items():
+  u=merge(ranges);nvout[name]={'count':len(ranges),'sum_s':duration(ranges),'union_s':duration(u),'gpu_overlap_s':overlap(u,merged)}
+ kernel_names=collections.defaultdict(lambda:[0,0])
+ for r in kerns:
+  name=strings.get(r['shortName'],'?');kernel_names[name][0]+=1;kernel_names[name][1]+=(r['end']-r['start'])/1e9
+ result={'file':Path(fn).name,'gpu_activity_span_s':span,'gpu_busy_union_s':duration(merged),'gpu_idle_within_span_s':span-duration(merged),'kernel_count':len(kerns),'graph_node_kernel_count':sum(bool(r['graphNodeId']) for r in kerns),'api':dict(sorted(api.items(),key=lambda x:-x[1][1])),'nvtx':nvout,'dispatch':dict(dispatch),'dispatch_timeline':dispatch_timeline,'top_kernels':sorted(kernel_names.items(),key=lambda x:-x[1][1])[:20],'notes':'GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs.'}
+ dest=Path(fn).with_suffix('.summary.json');dest.write_text(json.dumps(result,indent=2));print(dest)