import json,time,subprocess,os from pathlib import Path out=Path(__file__).resolve().parent low=[] critical=[] initial_oom=None while not (out/'monitor.stop').exists(): mem={l.split(':')[0]:int(l.split()[1]) for l in Path('/proc/meminfo').read_text().splitlines()} vm=dict(l.split() for l in Path('/proc/vmstat').read_text().splitlines()) state=subprocess.run(['docker','inspect','--format','{{json .State}}','qwen38-flash-vllm'],capture_output=True,text=True) status=json.loads(state.stdout) if state.returncode==0 else {} if initial_oom is None: initial_oom=int(vm.get('oom_kill',0)) swapped=(int(vm['pswpin'])+int(vm['pswpout']))*os.sysconf('SC_PAGE_SIZE') rec={'time':time.time(),'available_kib':mem['MemAvailable'],'swap_used_kib':mem['SwapTotal']-mem['SwapFree'], 'pswpin':int(vm['pswpin']),'pswpout':int(vm['pswpout']),'status':status.get('Status'), 'oom_killed':status.get('OOMKilled'),'host_oom_kills':int(vm.get('oom_kill',0)),'memory_pressure':Path('/proc/pressure/memory').read_text().strip()} print(json.dumps(rec),flush=True) if mem['MemAvailable']<2*1024*1024: critical.append(time.monotonic()) else: critical=[] if int(vm.get('oom_kill',0))>initial_oom or (len(critical)>=2 and critical[-1]-critical[0]>=5): (out/'GUARD_STOP').write_text('Host OOM counter increased or MemAvailable below 2 GiB for 5s') subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm']) break if mem['MemAvailable']<4*1024*1024: low.append((time.monotonic(),swapped)) else: low=[] if len(low)>=7 and low[-1][0]-low[0][0]>=30 and low[-1][1]-low[0][1]>512*1024**2: (out/'GUARD_STOP').write_text('MemAvailable below 4 GiB for 30s with >512 MiB swapping') subprocess.run(['docker','stop','-t','5','qwen38-flash-vllm']) break if status.get('OOMKilled'): (out/'GUARD_STOP').write_text('Docker reported OOMKilled') break time.sleep(5)