#!/bin/bash
# /usr/local/bin/dashcaddy-watchdog — self-healing guard for DashCaddy on DNS2.
#
# Heals four failure classes that have actually knocked DashCaddy down:
#   1. Container down/unhealthy/removed          -> docker start / bash start.sh
#   2. Rogue host node process stealing :3001    -> kill it, restart container
#   3. Caddyfile wiped by a foreign generator    -> restore known-good, restart caddy
#   4. Caddy down or not serving                 -> restart caddy
#
# Runs from dashcaddy-watchdog.service (systemd timer, every 30s) or manually.
# Alerts go to Telegram (Sami) on every corrective action; rate-limited per
# action class (default 15 min) to avoid spam during flapping. All state in
# /var/lib/dashcaddy-watchdog/ so restarts of the watchdog never flap.
#
# Exit codes: 0 = healthy or healed cleanly. Healing never exits non-zero —
# a permanently-failed unit would pollute systemctl --failed monitoring.

set -u

CONTAINER="dashcaddy-api"
START_SH="/opt/dashcaddy/start.sh"
CADDYFILE="/etc/caddy/Caddyfile"
KNOWN_GOOD="/var/lib/dashcaddy-watchdog/known-good-Caddyfile"
STATE_DIR="/var/lib/dashcaddy-watchdog"
LOG="/var/log/dashcaddy-watchdog.log"
ALERT_COOLDOWN=$((15 * 60))   # seconds between alerts of the same class
LOG_MAX=1048576               # 1 MiB
# Caddyfile integrity gates (adversarial review: markers alone can be
# satisfied by a damaged file). Size floor excludes the 4952-byte foreign-
# generator file from the 2026-08-13 incident; service-block floor excludes
# marker-duplication damage.
CADDYFILE_MIN_BYTES=10000
CADDYFILE_MIN_SITES=10

mkdir -p "$STATE_DIR"
touch "$LOG"

log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"; }

# --- Telegram alerting (best-effort; a notify failure never blocks healing) ---
notify() {  # notify <class> <message>
  local class="$1" msg="$2"
  local now last elapsed
  now=$(date +%s)
  local stamp="$STATE_DIR/alert-$class"
  if [ -f "$stamp" ]; then
    last=$(cat "$stamp" 2>/dev/null | tr -cd '0-9')
    last="${last:-0}"
    elapsed=$(( now - last ))
    if [ "$elapsed" -lt "$ALERT_COOLDOWN" ]; then
      log "ALERT-SUPPRESSED class=$class (${elapsed}s < ${ALERT_COOLDOWN}s)"
      return 0
    fi
  fi
  local token chat sent
  token=$(grep -E '^HERMES_ENV_TELEGRAM_BOT_TOKEN=' /root/.hermes/.env | head -1 | cut -d= -f2-)
  chat="637130179"
  if [ -z "$token" ]; then
    log "ALERT-NO-TOKEN class=$class msg=$msg"
    return 0
  fi
  sent=$(curl -s -m 10 -X POST "https://api.telegram.org/bot${token}/sendMessage" \
    -d chat_id="$chat" -d text="🛡️ DashCaddy watchdog (DNS2): $msg" \
    2>/dev/null | grep -c '"ok":true')
  if [ "${sent:-0}" -ge 1 ]; then
    date +%s > "$stamp"
    log "ALERT-SENT class=$class msg=$msg"
  else
    log "ALERT-SEND-FAILED class=$class (cooldown NOT burned)"
  fi
}

log_rotate() {
  [ -f "$LOG" ] || return 0
  local size
  size=$(stat -c%s "$LOG" 2>/dev/null || echo 0)
  if [ "$size" -gt "$LOG_MAX" ]; then
    tail -c $((LOG_MAX / 2)) "$LOG" > "${LOG}.tmp" && mv "${LOG}.tmp" "$LOG"
  fi
}

# --- Health probes -----------------------------------------------------------
container_running() {
  docker ps --filter "name=^${CONTAINER}$" --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER}$"
}

container_healthy() {
  local st started age
  st=$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null) || return 1
  if [ "$st" = "healthy" ]; then return 0; fi
  if [ "$st" = "starting" ]; then
    # Max-starting-age guard (adversarial review): never park forever on a
    # container stuck in 'starting'. StartPeriod is 10s; allow 90s slack.
    started=$(docker inspect -f '{{.State.StartedAt}}' "$CONTAINER" 2>/dev/null)
    age=$(( $(date +%s) - $(date -d "${started:-1970-01-01}" +%s 2>/dev/null || echo 0) ))
    [ "$age" -le 90 ]
    return
  fi
  return 1
}

rogue_on_port() {
  # any listener on 127.0.0.1:3001 that is NOT docker-proxy
  ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -q .
}

caddy_up() {
  systemctl is-active --quiet caddy
}

caddy_serving() {
  # status.sami via loopback; internal CA cert -> -k
  curl -sk -m 8 -o /dev/null -w '%{http_code}' --resolve status.sami:443:127.0.0.1 https://status.sami/ 2>/dev/null | grep -qE '^[23]'
}

caddyfile_intact() {
  # Integrity gates (adversarial review hardened): markers + size floor +
  # service-block floor + caddy syntax validation.
  [ -f "$CADDYFILE" ] || return 1
  local refs ca bytes sites
  refs=$(grep -c 'dashcaddy_auth' "$CADDYFILE" 2>/dev/null || echo 0)
  ca=$(grep -c 'sami-ca' "$CADDYFILE" 2>/dev/null || echo 0)
  bytes=$(stat -c%s "$CADDYFILE" 2>/dev/null || echo 0)
  sites=$(grep -cE '^[a-z0-9.-]+\.(sami|net|com|me|org)[^a-z0-9-]*\{$' "$CADDYFILE" 2>/dev/null || echo 0)
  [ "$refs" -ge 8 ] && [ "$ca" -ge 1 ] && [ "$bytes" -ge "$CADDYFILE_MIN_BYTES" ] && [ "$sites" -ge "$CADDYFILE_MIN_SITES" ]
}

refresh_known_good() {
  # Refuse to refresh within 10 minutes of a caddyfile heal (prevents the
  # partial-write / post-heal window from poisoning the snapshot).
  local heal_stamp="$STATE_DIR/last-caddyfile-heal"
  if [ -f "$heal_stamp" ]; then
    local since=$(( $(date +%s) - $(cat "$heal_stamp" | tr -cd '0-9' || echo 0) ))
    if [ "$since" -lt 600 ]; then
      return 0
    fi
  fi
  if caddyfile_intact; then
    if [ -f "$KNOWN_GOOD" ] && cmp -s "$CADDYFILE" "$KNOWN_GOOD"; then
      return 0  # unchanged — keep existing snapshot (preserves mtime)
    fi
    cp -a "$CADDYFILE" "$KNOWN_GOOD"
    log "KNOWN-GOOD refreshed ($(stat -c%s "$KNOWN_GOOD" 2>/dev/null || echo '?') bytes)"
  fi
}

# --- Remediation --------------------------------------------------------------
heal_container_start() {
  log "HEAL container-start"
  docker start "$CONTAINER" >/dev/null 2>&1
  if ! container_running; then
    log "HEAL container-start failed; falling back to start.sh"
    bash "$START_SH" >> "$LOG" 2>&1
  fi
  notify container "container was down — restarted it"
}

heal_container_recreate() {
  log "HEAL container-recreate (start.sh)"
  bash "$START_SH" >> "$LOG" 2>&1
  notify container "container unhealthy — recreated via start.sh"
}

heal_rogue_port() {
  local pids
  pids=$(ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -oP 'pid=\K[0-9]+' | sort -u)
  log "HEAL rogue-port pids=$pids"
  for pid in $pids; do
    [ "$pid" = "$$" ] && continue
    local cmdline
    cmdline=$(cat /proc/$pid/cmdline 2>/dev/null | tr '\0' ' ')
    case "$cmdline" in
      *docker-proxy*|*dashcaddy-watchdog*) continue ;;
    esac
    # PID recycling guard: the process must still be listening on :3001
    if ! ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -q "pid=$pid"; then
      continue
    fi
    log "KILL pid=$pid cmd=$cmdline"
    kill "$pid" 2>/dev/null || true
  done
  sleep 3
  ss -H -tlnp 'sport = :3001' | grep -q docker-proxy || docker restart "$CONTAINER" >/dev/null 2>&1
  notify rogue-port "rogue host process was holding port 3001 — killed it, container restarted"
}

heal_caddyfile() {
  log "HEAL caddyfile-restore"
  if [ -f "$KNOWN_GOOD" ]; then
    cp -a "$KNOWN_GOOD" "$CADDYFILE"
    chown caddy:caddy "$CADDYFILE" 2>/dev/null || true
    systemctl restart caddy
    notify caddyfile "Caddyfile was wiped/overwritten by a foreign generator — restored known-good and restarted Caddy"
  else
    notify caddyfile "Caddyfile damaged and no known-good snapshot exists — MANUAL ACTION NEEDED"
  fi
  # Post-heal grace stamp: caddy_serving() skipped for 60s after this point.
  date +%s > "$STATE_DIR/last-caddyfile-heal"
}

heal_caddy_down() {
  log "HEAL caddy-restart"
  systemctl restart caddy
  notify caddy "caddy was down — restarted it"
}

heal_caddy_5xx() {
  log "HEAL caddy-5xx"
  systemctl restart caddy
  notify caddy "caddy was not serving status.sami (non-2xx/3xx) — restarted it"
}

# --- Main ----------------------------------------------------------------------
log_rotate
ACTION_TAKEN=0

# 0. Maintain known-good Caddyfile snapshot whenever current file is intact.
refresh_known_good

# 3. Caddyfile integrity (before caddy health so restore happens first).
if ! caddyfile_intact && [ -f "$KNOWN_GOOD" ]; then
  heal_caddyfile
  ACTION_TAKEN=1
fi

# 4. Caddy up + serving
if ! caddy_up; then
  heal_caddy_down
  ACTION_TAKEN=1
fi
if ! caddy_serving; then
  # Skip if we JUST restarted caddy this cycle (post-heal grace).
  if [ -f "$STATE_DIR/last-caddyfile-heal" ]; then
    local_grace=$(( $(date +%s) - $(cat "$STATE_DIR/last-caddyfile-heal" | tr -cd '0-9' || echo 0) ))
    if [ "$local_grace" -lt 60 ]; then
      log "caddy_serving skipped (post-caddyfile-heal grace ${local_grace}s)"
    else
      heal_caddy_5xx
      ACTION_TAKEN=1
    fi
  else
    heal_caddy_5xx
    ACTION_TAKEN=1
  fi
fi

# 2. Rogue host process on :3001
if rogue_on_port; then
  heal_rogue_port
  ACTION_TAKEN=1
fi

# 1. Container down/unhealthy (last: start.sh recreates and re-binds 3001)
if ! container_running; then
  heal_container_start
  ACTION_TAKEN=1
elif ! container_healthy; then
  heal_container_recreate
  ACTION_TAKEN=1
fi

if [ "$ACTION_TAKEN" = "1" ]; then
  log "cycle complete: corrective action taken"
else
  log "cycle complete: healthy"
fi
exit 0