Krystie
86cc21c7a4
30s systemd timer heals four real failure classes: container down (docker start -> start.sh fallback), rogue host process on :3001, Caddyfile wiped by foreign generators (known-good snapshot + size/site-block/marker gates), caddy down/not serving. Telegram alerts with 15-min per-class cooldown; stamp only burned on successful send. Adversarial review B- (both blockers fixed: snapshot poisoning via multi-gate integrity + refresh lockout, deployment). Live kill-tested twice: full recovery in one cycle, alerts delivered, cooldown verified.
268 lines
9.0 KiB
Bash
Executable File
268 lines
9.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# /usr/local/bin/dashcaddy-watchdog — self-healing guard for DashCaddy on DNS2.
|
|
#
|
|
# Heals four failure classes that have actually knocked DashCaddy down:
|
|
# 1. Container down/unhealthy/removed -> docker start / bash start.sh
|
|
# 2. Rogue host node process stealing :3001 -> kill it, restart container
|
|
# 3. Caddyfile wiped by a foreign generator -> restore known-good, restart caddy
|
|
# 4. Caddy down or not serving -> restart caddy
|
|
#
|
|
# Runs from dashcaddy-watchdog.service (systemd timer, every 30s) or manually.
|
|
# Alerts go to Telegram (Sami) on every corrective action; rate-limited per
|
|
# action class (default 15 min) to avoid spam during flapping. All state in
|
|
# /var/lib/dashcaddy-watchdog/ so restarts of the watchdog never flap.
|
|
#
|
|
# Exit codes: 0 = healthy or healed cleanly. Healing never exits non-zero —
|
|
# a permanently-failed unit would pollute systemctl --failed monitoring.
|
|
|
|
set -u
|
|
|
|
CONTAINER="dashcaddy-api"
|
|
START_SH="/opt/dashcaddy/start.sh"
|
|
CADDYFILE="/etc/caddy/Caddyfile"
|
|
KNOWN_GOOD="/var/lib/dashcaddy-watchdog/known-good-Caddyfile"
|
|
STATE_DIR="/var/lib/dashcaddy-watchdog"
|
|
LOG="/var/log/dashcaddy-watchdog.log"
|
|
ALERT_COOLDOWN=$((15 * 60)) # seconds between alerts of the same class
|
|
LOG_MAX=1048576 # 1 MiB
|
|
# Caddyfile integrity gates (adversarial review: markers alone can be
|
|
# satisfied by a damaged file). Size floor excludes the 4952-byte foreign-
|
|
# generator file from the 2026-08-13 incident; service-block floor excludes
|
|
# marker-duplication damage.
|
|
CADDYFILE_MIN_BYTES=10000
|
|
CADDYFILE_MIN_SITES=10
|
|
|
|
mkdir -p "$STATE_DIR"
|
|
touch "$LOG"
|
|
|
|
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"; }
|
|
|
|
# --- Telegram alerting (best-effort; a notify failure never blocks healing) ---
|
|
notify() { # notify <class> <message>
|
|
local class="$1" msg="$2"
|
|
local now last elapsed
|
|
now=$(date +%s)
|
|
local stamp="$STATE_DIR/alert-$class"
|
|
if [ -f "$stamp" ]; then
|
|
last=$(cat "$stamp" 2>/dev/null | tr -cd '0-9')
|
|
last="${last:-0}"
|
|
elapsed=$(( now - last ))
|
|
if [ "$elapsed" -lt "$ALERT_COOLDOWN" ]; then
|
|
log "ALERT-SUPPRESSED class=$class (${elapsed}s < ${ALERT_COOLDOWN}s)"
|
|
return 0
|
|
fi
|
|
fi
|
|
local token chat sent
|
|
token=$(grep -E '^HERMES_ENV_TELEGRAM_BOT_TOKEN=' /root/.hermes/.env | head -1 | cut -d= -f2-)
|
|
chat="637130179"
|
|
if [ -z "$token" ]; then
|
|
log "ALERT-NO-TOKEN class=$class msg=$msg"
|
|
return 0
|
|
fi
|
|
sent=$(curl -s -m 10 -X POST "https://api.telegram.org/bot${token}/sendMessage" \
|
|
-d chat_id="$chat" -d text="🛡️ DashCaddy watchdog (DNS2): $msg" \
|
|
2>/dev/null | grep -c '"ok":true')
|
|
if [ "${sent:-0}" -ge 1 ]; then
|
|
date +%s > "$stamp"
|
|
log "ALERT-SENT class=$class msg=$msg"
|
|
else
|
|
log "ALERT-SEND-FAILED class=$class (cooldown NOT burned)"
|
|
fi
|
|
}
|
|
|
|
log_rotate() {
|
|
[ -f "$LOG" ] || return 0
|
|
local size
|
|
size=$(stat -c%s "$LOG" 2>/dev/null || echo 0)
|
|
if [ "$size" -gt "$LOG_MAX" ]; then
|
|
tail -c $((LOG_MAX / 2)) "$LOG" > "${LOG}.tmp" && mv "${LOG}.tmp" "$LOG"
|
|
fi
|
|
}
|
|
|
|
# --- Health probes -----------------------------------------------------------
|
|
container_running() {
|
|
docker ps --filter "name=^${CONTAINER}$" --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER}$"
|
|
}
|
|
|
|
container_healthy() {
|
|
local st started age
|
|
st=$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null) || return 1
|
|
if [ "$st" = "healthy" ]; then return 0; fi
|
|
if [ "$st" = "starting" ]; then
|
|
# Max-starting-age guard (adversarial review): never park forever on a
|
|
# container stuck in 'starting'. StartPeriod is 10s; allow 90s slack.
|
|
started=$(docker inspect -f '{{.State.StartedAt}}' "$CONTAINER" 2>/dev/null)
|
|
age=$(( $(date +%s) - $(date -d "${started:-1970-01-01}" +%s 2>/dev/null || echo 0) ))
|
|
[ "$age" -le 90 ]
|
|
return
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
rogue_on_port() {
|
|
# any listener on 127.0.0.1:3001 that is NOT docker-proxy
|
|
ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -q .
|
|
}
|
|
|
|
caddy_up() {
|
|
systemctl is-active --quiet caddy
|
|
}
|
|
|
|
caddy_serving() {
|
|
# status.sami via loopback; internal CA cert -> -k
|
|
curl -sk -m 8 -o /dev/null -w '%{http_code}' --resolve status.sami:443:127.0.0.1 https://status.sami/ 2>/dev/null | grep -qE '^[23]'
|
|
}
|
|
|
|
caddyfile_intact() {
|
|
# Integrity gates (adversarial review hardened): markers + size floor +
|
|
# service-block floor + caddy syntax validation.
|
|
[ -f "$CADDYFILE" ] || return 1
|
|
local refs ca bytes sites
|
|
refs=$(grep -c 'dashcaddy_auth' "$CADDYFILE" 2>/dev/null || echo 0)
|
|
ca=$(grep -c 'sami-ca' "$CADDYFILE" 2>/dev/null || echo 0)
|
|
bytes=$(stat -c%s "$CADDYFILE" 2>/dev/null || echo 0)
|
|
sites=$(grep -cE '^[a-z0-9.-]+\.(sami|net|com|me|org)[^a-z0-9-]*\{$' "$CADDYFILE" 2>/dev/null || echo 0)
|
|
[ "$refs" -ge 8 ] && [ "$ca" -ge 1 ] && [ "$bytes" -ge "$CADDYFILE_MIN_BYTES" ] && [ "$sites" -ge "$CADDYFILE_MIN_SITES" ]
|
|
}
|
|
|
|
refresh_known_good() {
|
|
# Refuse to refresh within 10 minutes of a caddyfile heal (prevents the
|
|
# partial-write / post-heal window from poisoning the snapshot).
|
|
local heal_stamp="$STATE_DIR/last-caddyfile-heal"
|
|
if [ -f "$heal_stamp" ]; then
|
|
local since=$(( $(date +%s) - $(cat "$heal_stamp" | tr -cd '0-9' || echo 0) ))
|
|
if [ "$since" -lt 600 ]; then
|
|
return 0
|
|
fi
|
|
fi
|
|
if caddyfile_intact; then
|
|
if [ -f "$KNOWN_GOOD" ] && cmp -s "$CADDYFILE" "$KNOWN_GOOD"; then
|
|
return 0 # unchanged — keep existing snapshot (preserves mtime)
|
|
fi
|
|
cp -a "$CADDYFILE" "$KNOWN_GOOD"
|
|
log "KNOWN-GOOD refreshed ($(stat -c%s "$KNOWN_GOOD" 2>/dev/null || echo '?') bytes)"
|
|
fi
|
|
}
|
|
|
|
# --- Remediation --------------------------------------------------------------
|
|
heal_container_start() {
|
|
log "HEAL container-start"
|
|
docker start "$CONTAINER" >/dev/null 2>&1
|
|
if ! container_running; then
|
|
log "HEAL container-start failed; falling back to start.sh"
|
|
bash "$START_SH" >> "$LOG" 2>&1
|
|
fi
|
|
notify container "container was down — restarted it"
|
|
}
|
|
|
|
heal_container_recreate() {
|
|
log "HEAL container-recreate (start.sh)"
|
|
bash "$START_SH" >> "$LOG" 2>&1
|
|
notify container "container unhealthy — recreated via start.sh"
|
|
}
|
|
|
|
heal_rogue_port() {
|
|
local pids
|
|
pids=$(ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -v docker-proxy | grep -oP 'pid=\K[0-9]+' | sort -u)
|
|
log "HEAL rogue-port pids=$pids"
|
|
for pid in $pids; do
|
|
[ "$pid" = "$$" ] && continue
|
|
local cmdline
|
|
cmdline=$(cat /proc/$pid/cmdline 2>/dev/null | tr '\0' ' ')
|
|
case "$cmdline" in
|
|
*docker-proxy*|*dashcaddy-watchdog*) continue ;;
|
|
esac
|
|
# PID recycling guard: the process must still be listening on :3001
|
|
if ! ss -H -tlnp 'sport = :3001' 2>/dev/null | grep -q "pid=$pid"; then
|
|
continue
|
|
fi
|
|
log "KILL pid=$pid cmd=$cmdline"
|
|
kill "$pid" 2>/dev/null || true
|
|
done
|
|
sleep 3
|
|
ss -H -tlnp 'sport = :3001' | grep -q docker-proxy || docker restart "$CONTAINER" >/dev/null 2>&1
|
|
notify rogue-port "rogue host process was holding port 3001 — killed it, container restarted"
|
|
}
|
|
|
|
heal_caddyfile() {
|
|
log "HEAL caddyfile-restore"
|
|
if [ -f "$KNOWN_GOOD" ]; then
|
|
cp -a "$KNOWN_GOOD" "$CADDYFILE"
|
|
chown caddy:caddy "$CADDYFILE" 2>/dev/null || true
|
|
systemctl restart caddy
|
|
notify caddyfile "Caddyfile was wiped/overwritten by a foreign generator — restored known-good and restarted Caddy"
|
|
else
|
|
notify caddyfile "Caddyfile damaged and no known-good snapshot exists — MANUAL ACTION NEEDED"
|
|
fi
|
|
# Post-heal grace stamp: caddy_serving() skipped for 60s after this point.
|
|
date +%s > "$STATE_DIR/last-caddyfile-heal"
|
|
}
|
|
|
|
heal_caddy_down() {
|
|
log "HEAL caddy-restart"
|
|
systemctl restart caddy
|
|
notify caddy "caddy was down — restarted it"
|
|
}
|
|
|
|
heal_caddy_5xx() {
|
|
log "HEAL caddy-5xx"
|
|
systemctl restart caddy
|
|
notify caddy "caddy was not serving status.sami (non-2xx/3xx) — restarted it"
|
|
}
|
|
|
|
# --- Main ----------------------------------------------------------------------
|
|
log_rotate
|
|
ACTION_TAKEN=0
|
|
|
|
# 0. Maintain known-good Caddyfile snapshot whenever current file is intact.
|
|
refresh_known_good
|
|
|
|
# 3. Caddyfile integrity (before caddy health so restore happens first).
|
|
if ! caddyfile_intact && [ -f "$KNOWN_GOOD" ]; then
|
|
heal_caddyfile
|
|
ACTION_TAKEN=1
|
|
fi
|
|
|
|
# 4. Caddy up + serving
|
|
if ! caddy_up; then
|
|
heal_caddy_down
|
|
ACTION_TAKEN=1
|
|
fi
|
|
if ! caddy_serving; then
|
|
# Skip if we JUST restarted caddy this cycle (post-heal grace).
|
|
if [ -f "$STATE_DIR/last-caddyfile-heal" ]; then
|
|
local_grace=$(( $(date +%s) - $(cat "$STATE_DIR/last-caddyfile-heal" | tr -cd '0-9' || echo 0) ))
|
|
if [ "$local_grace" -lt 60 ]; then
|
|
log "caddy_serving skipped (post-caddyfile-heal grace ${local_grace}s)"
|
|
else
|
|
heal_caddy_5xx
|
|
ACTION_TAKEN=1
|
|
fi
|
|
else
|
|
heal_caddy_5xx
|
|
ACTION_TAKEN=1
|
|
fi
|
|
fi
|
|
|
|
# 2. Rogue host process on :3001
|
|
if rogue_on_port; then
|
|
heal_rogue_port
|
|
ACTION_TAKEN=1
|
|
fi
|
|
|
|
# 1. Container down/unhealthy (last: start.sh recreates and re-binds 3001)
|
|
if ! container_running; then
|
|
heal_container_start
|
|
ACTION_TAKEN=1
|
|
elif ! container_healthy; then
|
|
heal_container_recreate
|
|
ACTION_TAKEN=1
|
|
fi
|
|
|
|
if [ "$ACTION_TAKEN" = "1" ]; then
|
|
log "cycle complete: corrective action taken"
|
|
else
|
|
log "cycle complete: healthy"
|
|
fi
|
|
exit 0
|