diff --git a/dashcaddy-api/scripts/dashcaddy-update.sh b/dashcaddy-api/scripts/dashcaddy-update.sh index 1d3f1b3..b90f411 100755 --- a/dashcaddy-api/scripts/dashcaddy-update.sh +++ b/dashcaddy-api/scripts/dashcaddy-update.sh @@ -26,6 +26,18 @@ readonly CHANNEL_CONF="${UPDATES_DIR}/channel.conf" readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data" readonly DATA_BACKUP_PREFIX="data-backup" +# Updater state (trigger.json / result.json) backup — keeps the audit trail +# (what version we were attempting, what the previous update's outcome was) tied +# to the same versioned backup directory as code + data. After a failed update, +# operators can inspect what was attempted without correlating timestamps, and +# rollback tooling can reconstruct a "what just happened" view of the update +# state machine. NOTE: we do NOT auto-restore trigger.json on rollback — the +# rollback handler reads a fresh trigger.json written by the operator/container; +# restoring the previous attempt's trigger would clobber the active rollback +# request. Backups here are read-only forensic evidence. +readonly UPDATE_STATE_BACKUP_PREFIX="update-state" +readonly TRIGGER_PROCESSING="${TRIGGER_FILE}.processing" + log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; } # Decide if a given release channel is acceptable on this host. @@ -112,6 +124,40 @@ backup_data_dir() { fi } +# ── Updater state backup (trigger.json.processing + result.json) ───────────── +# Captures what was being attempted + the last result so post-mortem can answer +# "why did this fail" without joining timestamps across files. Tolerates absent +# files (first-ever run) and locked files (chattr +i). Idempotent — re-running +# overwrites the previous backup. +backup_update_state() { + local backup_dir="$1" + local state_dir="${backup_dir}/${UPDATE_STATE_BACKUP_PREFIX}" + mkdir -p "$state_dir" + + local copied=0 + for src in "$TRIGGER_PROCESSING" "$RESULT_FILE"; do + if [[ -f "$src" ]]; then + # Unlock temporarily if immutable, copy, re-lock. + local was_locked=false + if lsattr -d "$src" 2>/dev/null | awk '{exit !($1 ~ /i/)}'; then + was_locked=true + chattr -i "$src" 2>/dev/null || true + fi + cp -f "$src" "${state_dir}/$(basename "$src")" 2>/dev/null && copied=$(( copied + 1 )) + if [[ "$was_locked" == "true" ]]; then + chattr +i "$src" 2>/dev/null || true + fi + fi + done + + if (( copied > 0 )); then + log "Update-state backup: ${copied} file(s) -> ${state_dir}" + else + log "Update-state backup: nothing to back up (no trigger/result files)" + rmdir "$state_dir" 2>/dev/null || true + fi +} + # ── Data restore ────────────────────────────────────────────────────────────── restore_data_dir() { local backup_dir="$1" @@ -329,6 +375,10 @@ main() { # Backup data/ directory (services.json, config.json, credentials, etc.) backup_data_dir "$backup_dir" + # Backup updater state (trigger.json.processing + result.json) so post-mortem + # has a forensic trail tied to this exact version's backup. + backup_update_state "$backup_dir" + cleanup_old_backups # 3. Copy new files from staging to API source diff --git a/dashcaddy-api/scripts/test-dashcaddy-update-backup.sh b/dashcaddy-api/scripts/test-dashcaddy-update-backup.sh new file mode 100755 index 0000000..1bc76c5 --- /dev/null +++ b/dashcaddy-api/scripts/test-dashcaddy-update-backup.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Regression test for dashcaddy-update.sh's backup_update_state() function. +# Run from the dashcaddy-api/scripts/ directory: +# bash test-dashcaddy-update-backup.sh +# Exit 0 = all assertions pass, non-zero = failure. + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly UPDATE_SCRIPT="${SCRIPT_DIR}/dashcaddy-update.sh" + +# Resolve the sibling copy if the local one is missing (the file lives at +# /root/dashcaddy/scripts/dashcaddy-update.sh AND dashcaddy-api/scripts/dashcaddy-update.sh +# and they're kept identical via `cp` during sprint work). +if [[ ! -f "$UPDATE_SCRIPT" ]]; then + UPDATE_SCRIPT="$(cd "${SCRIPT_DIR}/../../scripts" && pwd)/dashcaddy-update.sh" +fi + +if [[ ! -f "$UPDATE_SCRIPT" ]]; then + echo "FAIL: dashcaddy-update.sh not found at $UPDATE_SCRIPT" + exit 1 +fi + +# ── Test harness ────────────────────────────────────────────────────────────── +pass=0 +fail=0 + +assert_eq() { + local desc="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + echo " PASS: $desc" + pass=$(( pass + 1 )) + else + echo " FAIL: $desc — expected '$expected', got '$actual'" + fail=$(( fail + 1 )) + fi +} + +assert_file_contains() { + local desc="$1" file="$2" needle="$3" + if grep -q "$needle" "$file"; then + echo " PASS: $desc" + pass=$(( pass + 1 )) + else + echo " FAIL: $desc — '$needle' not in $file" + fail=$(( fail + 1 )) + fi +} + +# Extract backup_update_state() function body from the real script +EXTRACTED=$(awk '/^backup_update_state\(\) \{/,/^\}$/' "$UPDATE_SCRIPT") + +if [[ -z "$EXTRACTED" ]]; then + echo "FAIL: backup_update_state() function not found in $UPDATE_SCRIPT" + exit 1 +fi + +# ── Test 1: Both files present — function copies both ───────────────────────── +echo "=== Test 1: both trigger.json.processing + result.json present ===" +TMP=$(mktemp -d) +mkdir -p "$TMP/updates" +cat > "$TMP/trigger.json.processing" << 'EOF' +{"action":"update","version":"1.14.10","fromVersion":"1.14.8","channel":"stable","commit":"abc1234","stagingDir":"/opt/dashcaddy/updates/staging/dashcaddy-api","apiSourceDir":"/opt/dashcaddy/dashcaddy-api"} +EOF +cat > "$TMP/result.json" << 'EOF' +{"success":false,"version":"1.14.8","error":"Docker build failed","timestamp":"2026-07-10T12:34:56Z"} +EOF + +bash -c " + set -euo pipefail + TRIGGER_FILE='$TMP/trigger.json' + RESULT_FILE='$TMP/result.json' + TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\" + UPDATE_STATE_BACKUP_PREFIX='update-state' + log() { :; } + $EXTRACTED + backup_update_state '$TMP/backup/v1.14.8' +" + +assert_eq "backup dir created" "1" "$(find "$TMP/backup" -mindepth 1 -maxdepth 1 -type d | wc -l)" +assert_eq "update-state subdir exists" "1" "$(test -d "$TMP/backup/v1.14.8/update-state" && echo 1 || echo 0)" +assert_eq "trigger backup exists" "1" "$(test -f "$TMP/backup/v1.14.8/update-state/trigger.json.processing" && echo 1 || echo 0)" +assert_eq "result backup exists" "1" "$(test -f "$TMP/backup/v1.14.8/update-state/result.json" && echo 1 || echo 0)" +assert_file_contains "trigger content preserved" "$TMP/backup/v1.14.8/update-state/trigger.json.processing" '"fromVersion":"1.14.8"' +assert_file_contains "result content preserved" "$TMP/backup/v1.14.8/update-state/result.json" '"Docker build failed"' +assert_eq "trigger content byte-identical" "$(wc -c < "$TMP/trigger.json.processing")" "$(wc -c < "$TMP/backup/v1.14.8/update-state/trigger.json.processing")" +assert_eq "result content byte-identical" "$(wc -c < "$TMP/result.json")" "$(wc -c < "$TMP/backup/v1.14.8/update-state/result.json")" +rm -rf "$TMP" + +# ── Test 2: Only result.json exists — function copies only that ────────────── +echo "=== Test 2: only result.json present ===" +TMP=$(mktemp -d) +mkdir -p "$TMP/updates" +echo '{"success":true,"version":"1.14.7","duration":12}' > "$TMP/result.json" + +bash -c " + set -euo pipefail + TRIGGER_FILE='$TMP/trigger.json' + RESULT_FILE='$TMP/result.json' + TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\" + UPDATE_STATE_BACKUP_PREFIX='update-state' + log() { :; } + $EXTRACTED + backup_update_state '$TMP/backup/v1.14.7' +" + +assert_eq "trigger backup absent" "0" "$(test -f "$TMP/backup/v1.14.7/update-state/trigger.json.processing" 2>/dev/null && echo 1 || echo 0)" +assert_eq "result backup exists" "1" "$(test -f "$TMP/backup/v1.14.7/update-state/result.json" && echo 1 || echo 0)" +rm -rf "$TMP" + +# ── Test 3: No state files at all — function is a no-op (cleans up empty dir) ─ +echo "=== Test 3: no state files present ===" +TMP=$(mktemp -d) +mkdir -p "$TMP/updates" + +bash -c " + set -euo pipefail + TRIGGER_FILE='$TMP/trigger.json' + RESULT_FILE='$TMP/result.json' + TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\" + UPDATE_STATE_BACKUP_PREFIX='update-state' + log() { :; } + $EXTRACTED + backup_update_state '$TMP/backup/v1.14.6' +" + +assert_eq "no update-state dir when nothing to back up" "0" "$(test -d "$TMP/backup/v1.14.6/update-state" 2>/dev/null && echo 1 || echo 0)" +rm -rf "$TMP" + +# ── Test 4: Idempotency — running twice doesn't fail or accumulate ────────── +echo "=== Test 4: idempotency (run twice, no error, same single backup) ===" +TMP=$(mktemp -d) +mkdir -p "$TMP/updates" +echo '{"action":"update","version":"1.14.10","fromVersion":"1.14.8"}' > "$TMP/trigger.json.processing" + +bash -c " + set -euo pipefail + TRIGGER_FILE='$TMP/trigger.json' + RESULT_FILE='$TMP/result.json' + TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\" + UPDATE_STATE_BACKUP_PREFIX='update-state' + log() { :; } + $EXTRACTED + backup_update_state '$TMP/backup/v1.14.8' + backup_update_state '$TMP/backup/v1.14.8' +" 2>&1 | grep -E "(ERROR|FAIL)" || true + +assert_eq "only one trigger backup after 2 runs" "1" "$(find "$TMP/backup/v1.14.8/update-state" -name "trigger.json.processing" 2>/dev/null | wc -l)" +rm -rf "$TMP" + +# ── Test 5: main() calls backup_update_state in the right spot ────────────── +echo "=== Test 5: main() flow — backup_update_state called after backup_data_dir ===" +if grep -q 'backup_update_state "\$backup_dir"' "$UPDATE_SCRIPT"; then + echo " PASS: backup_update_state invoked from main()" + pass=$(( pass + 1 )) +else + echo " FAIL: backup_update_state not invoked from main()" + fail=$(( fail + 1 )) +fi + +# Verify ordering: backup_update_state must come AFTER backup_data_dir in main() +DATA_LINE=$(grep -n 'backup_data_dir "\$backup_dir"' "$UPDATE_SCRIPT" | head -1 | cut -d: -f1) +STATE_LINE=$(grep -n 'backup_update_state "\$backup_dir"' "$UPDATE_SCRIPT" | head -1 | cut -d: -f1) +if (( STATE_LINE > DATA_LINE )); then + echo " PASS: backup_update_state (line $STATE_LINE) called AFTER backup_data_dir (line $DATA_LINE)" + pass=$(( pass + 1 )) +else + echo " FAIL: backup_update_state (line $STATE_LINE) is NOT after backup_data_dir (line $DATA_LINE)" + fail=$(( fail + 1 )) +fi + +# ── Summary ────────────────────────────────────────────────────────────────── +echo +echo "═══════════════════════════════════════════" +echo " backup_update_state() regression test" +echo " PASS: $pass FAIL: $fail" +echo "═══════════════════════════════════════════" + +if (( fail > 0 )); then + exit 1 +fi +echo "All tests passed." diff --git a/scripts/dashcaddy-update.sh b/scripts/dashcaddy-update.sh index 1d3f1b3..b90f411 100755 --- a/scripts/dashcaddy-update.sh +++ b/scripts/dashcaddy-update.sh @@ -26,6 +26,18 @@ readonly CHANNEL_CONF="${UPDATES_DIR}/channel.conf" readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data" readonly DATA_BACKUP_PREFIX="data-backup" +# Updater state (trigger.json / result.json) backup — keeps the audit trail +# (what version we were attempting, what the previous update's outcome was) tied +# to the same versioned backup directory as code + data. After a failed update, +# operators can inspect what was attempted without correlating timestamps, and +# rollback tooling can reconstruct a "what just happened" view of the update +# state machine. NOTE: we do NOT auto-restore trigger.json on rollback — the +# rollback handler reads a fresh trigger.json written by the operator/container; +# restoring the previous attempt's trigger would clobber the active rollback +# request. Backups here are read-only forensic evidence. +readonly UPDATE_STATE_BACKUP_PREFIX="update-state" +readonly TRIGGER_PROCESSING="${TRIGGER_FILE}.processing" + log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; } # Decide if a given release channel is acceptable on this host. @@ -112,6 +124,40 @@ backup_data_dir() { fi } +# ── Updater state backup (trigger.json.processing + result.json) ───────────── +# Captures what was being attempted + the last result so post-mortem can answer +# "why did this fail" without joining timestamps across files. Tolerates absent +# files (first-ever run) and locked files (chattr +i). Idempotent — re-running +# overwrites the previous backup. +backup_update_state() { + local backup_dir="$1" + local state_dir="${backup_dir}/${UPDATE_STATE_BACKUP_PREFIX}" + mkdir -p "$state_dir" + + local copied=0 + for src in "$TRIGGER_PROCESSING" "$RESULT_FILE"; do + if [[ -f "$src" ]]; then + # Unlock temporarily if immutable, copy, re-lock. + local was_locked=false + if lsattr -d "$src" 2>/dev/null | awk '{exit !($1 ~ /i/)}'; then + was_locked=true + chattr -i "$src" 2>/dev/null || true + fi + cp -f "$src" "${state_dir}/$(basename "$src")" 2>/dev/null && copied=$(( copied + 1 )) + if [[ "$was_locked" == "true" ]]; then + chattr +i "$src" 2>/dev/null || true + fi + fi + done + + if (( copied > 0 )); then + log "Update-state backup: ${copied} file(s) -> ${state_dir}" + else + log "Update-state backup: nothing to back up (no trigger/result files)" + rmdir "$state_dir" 2>/dev/null || true + fi +} + # ── Data restore ────────────────────────────────────────────────────────────── restore_data_dir() { local backup_dir="$1" @@ -329,6 +375,10 @@ main() { # Backup data/ directory (services.json, config.json, credentials, etc.) backup_data_dir "$backup_dir" + # Backup updater state (trigger.json.processing + result.json) so post-mortem + # has a forensic trail tied to this exact version's backup. + backup_update_state "$backup_dir" + cleanup_old_backups # 3. Copy new files from staging to API source