docs(perf): profile CUDA graph coverage and PLE costs on Spark

2026-09-18 01:22:13 +08:00
parent f1a8964072
commit d7e1e745c3
23 changed files with 8384 additions and 1 deletions
@@ -0,0 +1,41 @@
+import json,time,subprocess,os
+from pathlib import Path
+out=Path(__file__).resolve().parent
+low=[]
+critical=[]
+initial_oom=None
+while not (out/'monitor.stop').exists():
+ mem={l.split(':')[0]:int(l.split()[1]) for l in Path('/proc/meminfo').read_text().splitlines()}
+ vm=dict(l.split() for l in Path('/proc/vmstat').read_text().splitlines())
+ state=subprocess.run(['docker','inspect','--format','{{json .State}}','qwen38-flash-vllm'],capture_output=True,text=True)
+ status=json.loads(state.stdout) if state.returncode==0 else {}
+ if initial_oom is None: initial_oom=int(vm.get('oom_kill',0))
+ swapped=(int(vm['pswpin'])+int(vm['pswpout']))*os.sysconf('SC_PAGE_SIZE')
+ rec={'time':time.time(),'available_kib':mem['MemAvailable'],'swap_used_kib':mem['SwapTotal']-mem['SwapFree'],
+      'pswpin':int(vm['pswpin']),'pswpout':int(vm['pswpout']),'status':status.get('Status'),
+      'oom_killed':status.get('OOMKilled'),'host_oom_kills':int(vm.get('oom_kill',0)),'memory_pressure':Path('/proc/pressure/memory').read_text().strip()}
+ print(json.dumps(rec),flush=True)
+ if mem['MemAvailable']<1024*1024:
+  (out/'GUARD_STOP').write_text('MemAvailable below 1 GiB')
+  subprocess.run(['docker','stop','-t','2','qwen38-flash-vllm'])
+  break
+ if mem['MemAvailable']<2*1024*1024:
+  critical.append(time.monotonic())
+ else:
+  critical=[]
+ if int(vm.get('oom_kill',0))>initial_oom or (len(critical)>=2 and critical[-1]-critical[0]>=5):
+  (out/'GUARD_STOP').write_text('Host OOM counter increased or MemAvailable below 2 GiB for 5s')
+  subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm'])
+  break
+ if mem['MemAvailable']<4*1024*1024:
+  low.append((time.monotonic(),swapped))
+ else:
+  low=[]
+ if len(low)>=7 and low[-1][0]-low[0][0]>=30 and low[-1][1]-low[0][1]>512*1024**2:
+  (out/'GUARD_STOP').write_text('MemAvailable below 4 GiB for 30s with >512 MiB swapping')
+  subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm'])
+  break
+ if status.get('OOMKilled'):
+  (out/'GUARD_STOP').write_text('Docker reported OOMKilled')
+  break
+ time.sleep(1)