#!/bin/bash
# Runs locally on the head. Stops Hy3 gracefully if free RAM enters the thrash
# zone, so vLLM overcommit can't wedge the box (SSH-dead) like last time.
# Exits 0 when API is up (guard no longer needed), 2 if it had to intervene.
LOG=/tmp/hy3-guard.log
THRESH_MB=6000
echo "$(date -u) guard start (thresh ${THRESH_MB}MB)" > "$LOG"
for i in $(seq 1 240); do
  cid=$(sg docker -c "docker ps -q --filter name=hy3" 2>/dev/null | head -1)
  if [ -z "$cid" ]; then echo "$(date -u) container gone, exiting" >> "$LOG"; exit 0; fi
  if curl -fsS --max-time 3 http://127.0.0.1:8888/v1/models >/dev/null 2>&1; then
    echo "$(date -u) API up, guard done" >> "$LOG"; exit 0
  fi
  avail=$(free -m | awk 'NR==2{print $7}')
  echo "$(date -u) avail=${avail}MB" >> "$LOG"
  if [ "${avail:-0}" -lt "$THRESH_MB" ]; then
    echo "$(date -u) !!! OOM GUARD TRIP avail=${avail}MB — stopping hy3 to protect the box" >> "$LOG"
    ( cd /home/jon/hy3-serve && bash stop-hy3.sh ) >> "$LOG" 2>&1
    exit 2
  fi
  sleep 5
done
echo "$(date -u) guard timeout" >> "$LOG"; exit 3