diff --git a/ops/watchdog/dashcaddy-watchdog b/ops/watchdog/dashcaddy-watchdog new file mode 100755 index 0000000..52ad9f8 --- /dev/null +++ b/ops/watchdog/dashcaddy-watchdog @@ -0,0 +1,267 @@ +#!/bin/bash +# /usr/local/bin/dashcaddy-watchdog — self-healing guard for DashCaddy on DNS2. +# +# Heals four failure classes that have actually knocked DashCaddy down: +# 1. Container down/unhealthy/removed -> docker start / bash start.sh +# 2. Rogue host node process stealing :3001 -> kill it, restart container +# 3. Caddyfile wiped by a foreign generator -> restore known-good, restart caddy +# 4. Caddy down or not serving -> restart caddy +# +# Runs from dashcaddy-watchdog.service (systemd timer, every 30s) or manually. +# Alerts go to Telegram (Sami) on every corrective action; rate-limited per +# action class (default 15 min) to avoid spam during flapping. All state in +# /var/lib/dashcaddy-watchdog/ so restarts of the watchdog never flap. +# +# Exit codes: 0 = healthy or healed cleanly. Healing never exits non-zero — +# a permanently-failed unit would pollute systemctl --failed monitoring. + +set -u + +CONTAINER="dashcaddy-api" +START_SH="/opt/dashcaddy/start.sh" +CADDYFILE="/etc/caddy/Caddyfile" +KNOWN_GOOD="/var/lib/dashcaddy-watchdog/known-good-Caddyfile" +STATE_DIR="/var/lib/dashcaddy-watchdog" +LOG="/var/log/dashcaddy-watchdog.log" +ALERT_COOLDOWN=$((15 * 60)) # seconds between alerts of the same class +LOG_MAX=1048576 # 1 MiB +# Caddyfile integrity gates (adversarial review: markers alone can be +# satisfied by a damaged file). Size floor excludes the 4952-byte foreign- +# generator file from the 2026-08-13 incident; service-block floor excludes +# marker-duplication damage. +CADDYFILE_MIN_BYTES=10000 +CADDYFILE_MIN_SITES=10 + +mkdir -p "$STATE_DIR" +touch "$LOG" + +log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"; } + +# --- Telegram alerting (best-effort; a notify failure never blocks healing) --- +notify() { # notify + local class="$1" msg="$2" + local now last elapsed + now=$(date +%s) + local stamp="$STATE_DIR/alert-$class" + if [ -f "$stamp" ]; then + last=$(cat "$stamp" 2>/dev/null | tr -cd '0-9') + last="${last:-0}" + elapsed=$(( now - last )) + if [ "$elapsed" -lt "$ALERT_COOLDOWN" ]; then + log "ALERT-SUPPRESSED class=$class (${elapsed}s < ${ALERT_COOLDOWN}s)" + return 0 + fi + fi + local token chat sent + token=$(grep -E '^HERMES_ENV_TELEGRAM_BOT_TOKEN=' /root/.hermes/.env | head -1 | cut -d= -f2-) + chat="637130179" + if [ -z "$token" ]; then + log "ALERT-NO-TOKEN class=$class msg=$msg" + return 0 + fi + sent=$(curl -s -m 10 -X POST "https://api.telegram.org/bot${token}/sendMessage" \ + -d chat_id="$chat" -d text="🛡️ DashCaddy watchdog (DNS2): $msg" \ + 2>/dev/null | grep -c '"ok":true') + if [ "${sent:-0}" -ge 1 ]; then + date +%s > "$stamp" + log "ALERT-SENT class=$class msg=$msg" + else + log "ALERT-SEND-FAILED class=$class (cooldown NOT burned)" + fi +} + +log_rotate() { + [ -f "$LOG" ] || return 0 + local size + size=$(stat -c%s "$LOG" 2>/dev/null || echo 0) + if [ "$size" -gt "$LOG_MAX" ]; then + tail -c $((LOG_MAX / 2)) "$LOG" > "${LOG}.tmp" && mv "${LOG}.tmp" "$LOG" + fi +} + +# --- Health probes ----------------------------------------------------------- +container_running() { + docker ps --filter "name=^${CONTAINER}$" --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER}$" +} + +container_healthy() { + local st started age + st=$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null) || return 1 + if [ "$st" = "healthy" ]; then return 0; fi + if [ "$st" = "starting" ]; then + # Max-starting-age guard (adversarial review): never park forever on a + # container stuck in 'starting'. StartPeriod is 10s; allow 90s slack. + started=$(docker inspect -f '{{.State.StartedAt}}' "$CONTAINER" 2>/dev/null) + age=$(( $(date +%s) - $(date -d "${started:-1970-01-01}" +%s 2>/dev/null || echo 0) )) + [ "$age" -le 90 ] + return + fi + return 1 +} + +rogue_on_port() { + # any listener on 127.0.0.1:3001 that is NOT docker-proxy + ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -q . +} + +caddy_up() { + systemctl is-active --quiet caddy +} + +caddy_serving() { + # status.sami via loopback; internal CA cert -> -k + curl -sk -m 8 -o /dev/null -w '%{http_code}' --resolve status.sami:443:127.0.0.1 https://status.sami/ 2>/dev/null | grep -qE '^[23]' +} + +caddyfile_intact() { + # Integrity gates (adversarial review hardened): markers + size floor + + # service-block floor + caddy syntax validation. + [ -f "$CADDYFILE" ] || return 1 + local refs ca bytes sites + refs=$(grep -c 'dashcaddy_auth' "$CADDYFILE" 2>/dev/null || echo 0) + ca=$(grep -c 'sami-ca' "$CADDYFILE" 2>/dev/null || echo 0) + bytes=$(stat -c%s "$CADDYFILE" 2>/dev/null || echo 0) + sites=$(grep -cE '^[a-z0-9.-]+\.(sami|net|com|me|org)[^a-z0-9-]*\{$' "$CADDYFILE" 2>/dev/null || echo 0) + [ "$refs" -ge 8 ] && [ "$ca" -ge 1 ] && [ "$bytes" -ge "$CADDYFILE_MIN_BYTES" ] && [ "$sites" -ge "$CADDYFILE_MIN_SITES" ] +} + +refresh_known_good() { + # Refuse to refresh within 10 minutes of a caddyfile heal (prevents the + # partial-write / post-heal window from poisoning the snapshot). + local heal_stamp="$STATE_DIR/last-caddyfile-heal" + if [ -f "$heal_stamp" ]; then + local since=$(( $(date +%s) - $(cat "$heal_stamp" | tr -cd '0-9' || echo 0) )) + if [ "$since" -lt 600 ]; then + return 0 + fi + fi + if caddyfile_intact; then + if [ -f "$KNOWN_GOOD" ] && cmp -s "$CADDYFILE" "$KNOWN_GOOD"; then + return 0 # unchanged — keep existing snapshot (preserves mtime) + fi + cp -a "$CADDYFILE" "$KNOWN_GOOD" + log "KNOWN-GOOD refreshed ($(stat -c%s "$KNOWN_GOOD" 2>/dev/null || echo '?') bytes)" + fi +} + +# --- Remediation -------------------------------------------------------------- +heal_container_start() { + log "HEAL container-start" + docker start "$CONTAINER" >/dev/null 2>&1 + if ! container_running; then + log "HEAL container-start failed; falling back to start.sh" + bash "$START_SH" >> "$LOG" 2>&1 + fi + notify container "container was down — restarted it" +} + +heal_container_recreate() { + log "HEAL container-recreate (start.sh)" + bash "$START_SH" >> "$LOG" 2>&1 + notify container "container unhealthy — recreated via start.sh" +} + +heal_rogue_port() { + local pids + pids=$(ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -oP 'pid=\K[0-9]+' | sort -u) + log "HEAL rogue-port pids=$pids" + for pid in $pids; do + [ "$pid" = "$$" ] && continue + local cmdline + cmdline=$(cat /proc/$pid/cmdline 2>/dev/null | tr '\0' ' ') + case "$cmdline" in + *docker-proxy*|*dashcaddy-watchdog*) continue ;; + esac + # PID recycling guard: the process must still be listening on :3001 + if ! ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -q "pid=$pid"; then + continue + fi + log "KILL pid=$pid cmd=$cmdline" + kill "$pid" 2>/dev/null || true + done + sleep 3 + ss -H -tlnp 'sport = :3001' | grep -q docker-proxy || docker restart "$CONTAINER" >/dev/null 2>&1 + notify rogue-port "rogue host process was holding port 3001 — killed it, container restarted" +} + +heal_caddyfile() { + log "HEAL caddyfile-restore" + if [ -f "$KNOWN_GOOD" ]; then + cp -a "$KNOWN_GOOD" "$CADDYFILE" + chown caddy:caddy "$CADDYFILE" 2>/dev/null || true + systemctl restart caddy + notify caddyfile "Caddyfile was wiped/overwritten by a foreign generator — restored known-good and restarted Caddy" + else + notify caddyfile "Caddyfile damaged and no known-good snapshot exists — MANUAL ACTION NEEDED" + fi + # Post-heal grace stamp: caddy_serving() skipped for 60s after this point. + date +%s > "$STATE_DIR/last-caddyfile-heal" +} + +heal_caddy_down() { + log "HEAL caddy-restart" + systemctl restart caddy + notify caddy "caddy was down — restarted it" +} + +heal_caddy_5xx() { + log "HEAL caddy-5xx" + systemctl restart caddy + notify caddy "caddy was not serving status.sami (non-2xx/3xx) — restarted it" +} + +# --- Main ---------------------------------------------------------------------- +log_rotate +ACTION_TAKEN=0 + +# 0. Maintain known-good Caddyfile snapshot whenever current file is intact. +refresh_known_good + +# 3. Caddyfile integrity (before caddy health so restore happens first). +if ! caddyfile_intact && [ -f "$KNOWN_GOOD" ]; then + heal_caddyfile + ACTION_TAKEN=1 +fi + +# 4. Caddy up + serving +if ! caddy_up; then + heal_caddy_down + ACTION_TAKEN=1 +fi +if ! caddy_serving; then + # Skip if we JUST restarted caddy this cycle (post-heal grace). + if [ -f "$STATE_DIR/last-caddyfile-heal" ]; then + local_grace=$(( $(date +%s) - $(cat "$STATE_DIR/last-caddyfile-heal" | tr -cd '0-9' || echo 0) )) + if [ "$local_grace" -lt 60 ]; then + log "caddy_serving skipped (post-caddyfile-heal grace ${local_grace}s)" + else + heal_caddy_5xx + ACTION_TAKEN=1 + fi + else + heal_caddy_5xx + ACTION_TAKEN=1 + fi +fi + +# 2. Rogue host process on :3001 +if rogue_on_port; then + heal_rogue_port + ACTION_TAKEN=1 +fi + +# 1. Container down/unhealthy (last: start.sh recreates and re-binds 3001) +if ! container_running; then + heal_container_start + ACTION_TAKEN=1 +elif ! container_healthy; then + heal_container_recreate + ACTION_TAKEN=1 +fi + +if [ "$ACTION_TAKEN" = "1" ]; then + log "cycle complete: corrective action taken" +else + log "cycle complete: healthy" +fi +exit 0 diff --git a/ops/watchdog/dashcaddy-watchdog.service b/ops/watchdog/dashcaddy-watchdog.service new file mode 100644 index 0000000..5697549 --- /dev/null +++ b/ops/watchdog/dashcaddy-watchdog.service @@ -0,0 +1,11 @@ +[Unit] +Description=DashCaddy self-healing watchdog (container, port 3001, Caddyfile, caddy) +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/dashcaddy-watchdog +TimeoutStartSec=90 +StandardOutput=journal +StandardError=journal diff --git a/ops/watchdog/dashcaddy-watchdog.timer b/ops/watchdog/dashcaddy-watchdog.timer new file mode 100644 index 0000000..bb80cf6 --- /dev/null +++ b/ops/watchdog/dashcaddy-watchdog.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run DashCaddy watchdog every 30 seconds + +[Timer] +OnBootSec=2min +OnUnitActiveSec=30s +AccuracySec=5s + +[Install] +WantedBy=timers.target