#!/usr/bin/env bash
# hy3-watchdog: poll vLLM /health and self-heal the 2-node Hy3 stack
# (worker-first) if the server dies. Logs to /home/jon/hy3-serve/watchdog.log.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8888/health}"
POLL_INTERVAL="${POLL_INTERVAL:-30}"
FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"
COOLDOWN="${COOLDOWN:-900}"   # min secs between restarts (init ~11 min)
LOG="${LOG:-$SCRIPT_DIR/watchdog.log}"
fails=0; last_restart=0
log(){ echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "$LOG" >&2; }
log "watchdog start (health=$HEALTH_URL threshold=$FAIL_THRESHOLD)"
while true; do
  if curl -fsS --max-time 10 "$HEALTH_URL" >/dev/null 2>&1; then
    [ "$fails" -ne 0 ] && log "health recovered after $fails fail(s)"
    fails=0
  else
    fails=$((fails+1))
    log "/health DOWN (consecutive=$fails/$FAIL_THRESHOLD)"
    if [ "$fails" -ge "$FAIL_THRESHOLD" ]; then
      now=$(date +%s)
      if [ $((now - last_restart)) -lt "$COOLDOWN" ]; then
        log "in cooldown ($((now-last_restart))s < ${COOLDOWN}s); skip"
      else
        log "RESTARTING Hy3 stack (worker-first)"
        bash "$SCRIPT_DIR/stop-hy3.sh"  >>"$LOG" 2>&1 || true
        bash "$SCRIPT_DIR/start-hy3.sh" >>"$LOG" 2>&1 || true
        ( nohup bash "$SCRIPT_DIR/oom-guard.sh" >/dev/null 2>&1 & ) || true
        last_restart=$(date +%s); fails=0
      fi
    fi
  fi
  sleep "$POLL_INTERVAL"
done