23 lines
1.0 KiB
Python
23 lines
1.0 KiB
Python
"""Log host/GPU headroom; stop the test server on host OOM or low RAM."""
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
def kv(path):
|
|
return {s.split()[0].rstrip(':'): int(s.split()[1]) for s in Path(path).read_text().splitlines() if len(s.split())>=2}
|
|
|
|
baseline = kv('/proc/vmstat')['oom_kill']
|
|
while True:
|
|
mem = kv('/proc/meminfo'); vm = kv('/proc/vmstat')
|
|
gpu = subprocess.run(['nvidia-smi','--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu',
|
|
'--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=10)
|
|
low = mem['MemAvailable'] < 8*1024*1024
|
|
oom = vm['oom_kill'] > baseline
|
|
print(json.dumps(dict(time=time.time(),available_gib=mem['MemAvailable']/1024**2,
|
|
oom_kill=vm['oom_kill'],gpu=gpu.stdout.strip(),stop=low or oom)),flush=True)
|
|
if low or oom:
|
|
subprocess.run(['docker','stop','-t','10','qwen38-flash-6000d'],timeout=30)
|
|
raise SystemExit('Host memory safeguard stopped server')
|
|
time.sleep(2)
|