docs(perf): profile CUDA graph coverage and PLE costs on Spark

experiments/profile128k/summarize_trace.py

@@ -0,0 +1,52 @@
+"""Export only CUDA/NVTX aggregates; omit process arguments and environment."""
+import sqlite3,json,sys,collections
+from pathlib import Path
+
+def merge(intervals):
+ out=[]
+ for a,b in sorted(intervals):
+  if b<=a:continue
+  if out and a<=out[-1][1]:out[-1]=(out[-1][0],max(out[-1][1],b))
+  else:out.append((a,b))
+ return out
+
+def duration(xs):return sum(b-a for a,b in xs)/1e9
+
+def overlap(a,b):
+ i=j=0;total=0
+ while i<len(a) and j<len(b):
+  total+=max(0,min(a[i][1],b[j][1])-max(a[i][0],b[j][0]))
+  if a[i][1]<b[j][1]:i+=1
+  else:j+=1
+ return total/1e9
+
+for fn in sys.argv[1:]:
+ db=sqlite3.connect(fn);db.row_factory=sqlite3.Row
+ tables={r[0] for r in db.execute("select name from sqlite_master where type='table'")}
+ strings=dict(db.execute('select id,value from StringIds'))
+ gpu=[];kerns=[];api=collections.defaultdict(lambda:[0,0]);nv=collections.defaultdict(list);dispatch=collections.Counter();dispatch_timeline=[]
+ for table in ['CUPTI_ACTIVITY_KIND_KERNEL','CUPTI_ACTIVITY_KIND_MEMCPY','CUPTI_ACTIVITY_KIND_MEMSET']:
+  if table in tables:
+   rows=list(db.execute('select * from '+table));gpu.extend((r['start'],r['end']) for r in rows)
+   if table.endswith('KERNEL'):kerns=rows
+ for table in ['CUPTI_ACTIVITY_KIND_RUNTIME','CUPTI_ACTIVITY_KIND_DRIVER']:
+  if table in tables:
+   for r in db.execute('select start,end,nameId from '+table):
+    name=strings.get(r['nameId'],str(r['nameId']));api[name][0]+=1;api[name][1]+=(r['end']-r['start'])/1e9
+ if 'NVTX_EVENTS' in tables:
+  for r in db.execute('select * from NVTX_EVENTS'):
+   keys=r.keys();name=r['text'] if 'text' in keys else None
+   if not name and 'textId' in keys:name=strings.get(r['textId'],'')
+   name=name or ''
+   if name.startswith('CG_DISPATCH:'):
+    dispatch[name]+=1;dispatch_timeline.append({'start_ns':r['start'],'name':name})
+   if name.startswith('PLE_') and r['end'] is not None:nv[name].append((r['start'],r['end']))
+ merged=merge(gpu);span=(merged[-1][1]-merged[0][0])/1e9 if merged else 0
+ nvout={}
+ for name,ranges in nv.items():
+  u=merge(ranges);nvout[name]={'count':len(ranges),'sum_s':duration(ranges),'union_s':duration(u),'gpu_overlap_s':overlap(u,merged)}
+ kernel_names=collections.defaultdict(lambda:[0,0])
+ for r in kerns:
+  name=strings.get(r['shortName'],'?');kernel_names[name][0]+=1;kernel_names[name][1]+=(r['end']-r['start'])/1e9
+ result={'file':Path(fn).name,'gpu_activity_span_s':span,'gpu_busy_union_s':duration(merged),'gpu_idle_within_span_s':span-duration(merged),'kernel_count':len(kerns),'graph_node_kernel_count':sum(bool(r['graphNodeId']) for r in kerns),'api':dict(sorted(api.items(),key=lambda x:-x[1][1])),'nvtx':nvout,'dispatch':dict(dispatch),'dispatch_timeline':dispatch_timeline,'top_kernels':sorted(kernel_names.items(),key=lambda x:-x[1][1])[:20],'notes':'GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs.'}
+ dest=Path(fn).with_suffix('.summary.json');dest.write_text(json.dumps(result,indent=2));print(dest)