"""Log host/GPU headroom; stop the test server on host OOM or low RAM.""" import json import subprocess import time from pathlib import Path def kv(path): return {s.split()[0].rstrip(':'): int(s.split()[1]) for s in Path(path).read_text().splitlines() if len(s.split())>=2} baseline = kv('/proc/vmstat')['oom_kill'] while True: mem = kv('/proc/meminfo'); vm = kv('/proc/vmstat') gpu = subprocess.run(['nvidia-smi','--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu', '--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=10) low = mem['MemAvailable'] < 8*1024*1024 oom = vm['oom_kill'] > baseline print(json.dumps(dict(time=time.time(),available_gib=mem['MemAvailable']/1024**2, oom_kill=vm['oom_kill'],gpu=gpu.stdout.strip(),stop=low or oom)),flush=True) if low or oom: subprocess.run(['docker','stop','-t','10','qwen38-flash-6000d'],timeout=30) raise SystemExit('Host memory safeguard stopped server') time.sleep(2)