27 lines
1.3 KiB
Python
27 lines
1.3 KiB
Python
import json,time,subprocess,os
|
|
from pathlib import Path
|
|
out=Path(__file__).resolve().parent
|
|
low=[]
|
|
while not (out/'monitor.stop').exists():
|
|
mem={l.split(':')[0]:int(l.split()[1]) for l in Path('/proc/meminfo').read_text().splitlines()}
|
|
vm=dict(l.split() for l in Path('/proc/vmstat').read_text().splitlines())
|
|
state=subprocess.run(['docker','inspect','--format','{{json .State}}','qwen38-flash-vllm'],capture_output=True,text=True)
|
|
status=json.loads(state.stdout) if state.returncode==0 else {}
|
|
swapped=(int(vm['pswpin'])+int(vm['pswpout']))*os.sysconf('SC_PAGE_SIZE')
|
|
rec={'time':time.time(),'available_kib':mem['MemAvailable'],'swap_used_kib':mem['SwapTotal']-mem['SwapFree'],
|
|
'pswpin':int(vm['pswpin']),'pswpout':int(vm['pswpout']),'status':status.get('Status'),
|
|
'oom_killed':status.get('OOMKilled'),'memory_pressure':Path('/proc/pressure/memory').read_text().strip()}
|
|
print(json.dumps(rec),flush=True)
|
|
if mem['MemAvailable']<4*1024*1024:
|
|
low.append((time.monotonic(),swapped))
|
|
else:
|
|
low=[]
|
|
if len(low)>=7 and low[-1][0]-low[0][0]>=30 and low[-1][1]-low[0][1]>512*1024**2:
|
|
(out/'GUARD_STOP').write_text('MemAvailable below 4 GiB for 30s with >512 MiB swapping')
|
|
subprocess.run(['docker','stop','-t','30','qwen38-flash-vllm'])
|
|
break
|
|
if status.get('OOMKilled'):
|
|
(out/'GUARD_STOP').write_text('Docker reported OOMKilled')
|
|
break
|
|
time.sleep(5)
|