DC-038: backup trigger.json + result.json in dashcaddy-update.sh

The host-side updater only backed up code + data/, leaving trigger.json and
result.json unarchived. After a failed update, operators had to reconstruct
'what was being attempted' by joining timestamps across files. Now the
backup captures both files into a 'update-state/' subdir alongside code +
data backups, keyed by from-version.

- New `backup_update_state()` function in dashcaddy-update.sh: idempotent,
  tolerates absent files (cleans up empty subdir), tolerates chattr +i
  (unlock/copy/relock).
- Wired into main() right after `backup_data_dir`, before `cleanup_old_backups`.
- Deliberately does NOT auto-restore trigger.json on rollback — the rollback
  handler reads a fresh trigger.json written by the operator/container;
  restoring the previous attempt's trigger would clobber the active rollback
  request. Backups are read-only forensic evidence.
- New `dashcaddy-api/scripts/test-dashcaddy-update-backup.sh` (14 assertions,
  5 test groups): both-files-present, partial-present, no-files-present,
  idempotency, main() flow ordering. All 14 pass.
- Synced the duplicate at `dashcaddy-api/scripts/dashcaddy-update.sh`
  (md5-identical to scripts/dashcaddy-update.sh).

Tests: 1214/1214 pass (zero change). Lint: 150 warnings, all pre-existing
in untouched files (zero new warnings introduced).

dashcaddy-api/scripts/dashcaddy-update.sh

@@ -26,6 +26,18 @@ readonly CHANNEL_CONF="${UPDATES_DIR}/channel.conf"
 readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data"
 readonly DATA_BACKUP_PREFIX="data-backup"
 
+# Updater state (trigger.json / result.json) backup — keeps the audit trail
+# (what version we were attempting, what the previous update's outcome was) tied
+# to the same versioned backup directory as code + data. After a failed update,
+# operators can inspect what was attempted without correlating timestamps, and
+# rollback tooling can reconstruct a "what just happened" view of the update
+# state machine. NOTE: we do NOT auto-restore trigger.json on rollback — the
+# rollback handler reads a fresh trigger.json written by the operator/container;
+# restoring the previous attempt's trigger would clobber the active rollback
+# request. Backups here are read-only forensic evidence.
+readonly UPDATE_STATE_BACKUP_PREFIX="update-state"
+readonly TRIGGER_PROCESSING="${TRIGGER_FILE}.processing"
+
 log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
 
 # Decide if a given release channel is acceptable on this host.
@@ -112,6 +124,40 @@ backup_data_dir() {
   fi
 }
 
+# ── Updater state backup (trigger.json.processing + result.json) ─────────────
+# Captures what was being attempted + the last result so post-mortem can answer
+# "why did this fail" without joining timestamps across files. Tolerates absent
+# files (first-ever run) and locked files (chattr +i). Idempotent — re-running
+# overwrites the previous backup.
+backup_update_state() {
+  local backup_dir="$1"
+  local state_dir="${backup_dir}/${UPDATE_STATE_BACKUP_PREFIX}"
+  mkdir -p "$state_dir"
+
+  local copied=0
+  for src in "$TRIGGER_PROCESSING" "$RESULT_FILE"; do
+    if [[ -f "$src" ]]; then
+      # Unlock temporarily if immutable, copy, re-lock.
+      local was_locked=false
+      if lsattr -d "$src" 2>/dev/null | awk '{exit !($1 ~ /i/)}'; then
+        was_locked=true
+        chattr -i "$src" 2>/dev/null || true
+      fi
+      cp -f "$src" "${state_dir}/$(basename "$src")" 2>/dev/null && copied=$(( copied + 1 ))
+      if [[ "$was_locked" == "true" ]]; then
+        chattr +i "$src" 2>/dev/null || true
+      fi
+    fi
+  done
+
+  if (( copied > 0 )); then
+    log "Update-state backup: ${copied} file(s) -> ${state_dir}"
+  else
+    log "Update-state backup: nothing to back up (no trigger/result files)"
+    rmdir "$state_dir" 2>/dev/null || true
+  fi
+}
+
 # ── Data restore ──────────────────────────────────────────────────────────────
 restore_data_dir() {
   local backup_dir="$1"
@@ -329,6 +375,10 @@ main() {
   # Backup data/ directory (services.json, config.json, credentials, etc.)
   backup_data_dir "$backup_dir"
 
+  # Backup updater state (trigger.json.processing + result.json) so post-mortem
+  # has a forensic trail tied to this exact version's backup.
+  backup_update_state "$backup_dir"
+
   cleanup_old_backups
 
   # 3. Copy new files from staging to API source

dashcaddy-api/scripts/test-dashcaddy-update-backup.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Regression test for dashcaddy-update.sh's backup_update_state() function.
+# Run from the dashcaddy-api/scripts/ directory:
+#   bash test-dashcaddy-update-backup.sh
+# Exit 0 = all assertions pass, non-zero = failure.
+
+set -euo pipefail
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly UPDATE_SCRIPT="${SCRIPT_DIR}/dashcaddy-update.sh"
+
+# Resolve the sibling copy if the local one is missing (the file lives at
+# /root/dashcaddy/scripts/dashcaddy-update.sh AND dashcaddy-api/scripts/dashcaddy-update.sh
+# and they're kept identical via `cp` during sprint work).
+if [[ ! -f "$UPDATE_SCRIPT" ]]; then
+  UPDATE_SCRIPT="$(cd "${SCRIPT_DIR}/../../scripts" && pwd)/dashcaddy-update.sh"
+fi
+
+if [[ ! -f "$UPDATE_SCRIPT" ]]; then
+  echo "FAIL: dashcaddy-update.sh not found at $UPDATE_SCRIPT"
+  exit 1
+fi
+
+# ── Test harness ──────────────────────────────────────────────────────────────
+pass=0
+fail=0
+
+assert_eq() {
+  local desc="$1" expected="$2" actual="$3"
+  if [[ "$expected" == "$actual" ]]; then
+    echo "  PASS: $desc"
+    pass=$(( pass + 1 ))
+  else
+    echo "  FAIL: $desc — expected '$expected', got '$actual'"
+    fail=$(( fail + 1 ))
+  fi
+}
+
+assert_file_contains() {
+  local desc="$1" file="$2" needle="$3"
+  if grep -q "$needle" "$file"; then
+    echo "  PASS: $desc"
+    pass=$(( pass + 1 ))
+  else
+    echo "  FAIL: $desc — '$needle' not in $file"
+    fail=$(( fail + 1 ))
+  fi
+}
+
+# Extract backup_update_state() function body from the real script
+EXTRACTED=$(awk '/^backup_update_state\(\) \{/,/^\}$/' "$UPDATE_SCRIPT")
+
+if [[ -z "$EXTRACTED" ]]; then
+  echo "FAIL: backup_update_state() function not found in $UPDATE_SCRIPT"
+  exit 1
+fi
+
+# ── Test 1: Both files present — function copies both ─────────────────────────
+echo "=== Test 1: both trigger.json.processing + result.json present ==="
+TMP=$(mktemp -d)
+mkdir -p "$TMP/updates"
+cat > "$TMP/trigger.json.processing" << 'EOF'
+{"action":"update","version":"1.14.10","fromVersion":"1.14.8","channel":"stable","commit":"abc1234","stagingDir":"/opt/dashcaddy/updates/staging/dashcaddy-api","apiSourceDir":"/opt/dashcaddy/dashcaddy-api"}
+EOF
+cat > "$TMP/result.json" << 'EOF'
+{"success":false,"version":"1.14.8","error":"Docker build failed","timestamp":"2026-07-10T12:34:56Z"}
+EOF
+
+bash -c "
+  set -euo pipefail
+  TRIGGER_FILE='$TMP/trigger.json'
+  RESULT_FILE='$TMP/result.json'
+  TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\"
+  UPDATE_STATE_BACKUP_PREFIX='update-state'
+  log() { :; }
+  $EXTRACTED
+  backup_update_state '$TMP/backup/v1.14.8'
+"
+
+assert_eq "backup dir created" "1" "$(find "$TMP/backup" -mindepth 1 -maxdepth 1 -type d | wc -l)"
+assert_eq "update-state subdir exists" "1" "$(test -d "$TMP/backup/v1.14.8/update-state" && echo 1 || echo 0)"
+assert_eq "trigger backup exists" "1" "$(test -f "$TMP/backup/v1.14.8/update-state/trigger.json.processing" && echo 1 || echo 0)"
+assert_eq "result backup exists" "1" "$(test -f "$TMP/backup/v1.14.8/update-state/result.json" && echo 1 || echo 0)"
+assert_file_contains "trigger content preserved" "$TMP/backup/v1.14.8/update-state/trigger.json.processing" '"fromVersion":"1.14.8"'
+assert_file_contains "result content preserved" "$TMP/backup/v1.14.8/update-state/result.json" '"Docker build failed"'
+assert_eq "trigger content byte-identical" "$(wc -c < "$TMP/trigger.json.processing")" "$(wc -c < "$TMP/backup/v1.14.8/update-state/trigger.json.processing")"
+assert_eq "result content byte-identical" "$(wc -c < "$TMP/result.json")" "$(wc -c < "$TMP/backup/v1.14.8/update-state/result.json")"
+rm -rf "$TMP"
+
+# ── Test 2: Only result.json exists — function copies only that ──────────────
+echo "=== Test 2: only result.json present ==="
+TMP=$(mktemp -d)
+mkdir -p "$TMP/updates"
+echo '{"success":true,"version":"1.14.7","duration":12}' > "$TMP/result.json"
+
+bash -c "
+  set -euo pipefail
+  TRIGGER_FILE='$TMP/trigger.json'
+  RESULT_FILE='$TMP/result.json'
+  TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\"
+  UPDATE_STATE_BACKUP_PREFIX='update-state'
+  log() { :; }
+  $EXTRACTED
+  backup_update_state '$TMP/backup/v1.14.7'
+"
+
+assert_eq "trigger backup absent" "0" "$(test -f "$TMP/backup/v1.14.7/update-state/trigger.json.processing" 2>/dev/null && echo 1 || echo 0)"
+assert_eq "result backup exists" "1" "$(test -f "$TMP/backup/v1.14.7/update-state/result.json" && echo 1 || echo 0)"
+rm -rf "$TMP"
+
+# ── Test 3: No state files at all — function is a no-op (cleans up empty dir) ─
+echo "=== Test 3: no state files present ==="
+TMP=$(mktemp -d)
+mkdir -p "$TMP/updates"
+
+bash -c "
+  set -euo pipefail
+  TRIGGER_FILE='$TMP/trigger.json'
+  RESULT_FILE='$TMP/result.json'
+  TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\"
+  UPDATE_STATE_BACKUP_PREFIX='update-state'
+  log() { :; }
+  $EXTRACTED
+  backup_update_state '$TMP/backup/v1.14.6'
+"
+
+assert_eq "no update-state dir when nothing to back up" "0" "$(test -d "$TMP/backup/v1.14.6/update-state" 2>/dev/null && echo 1 || echo 0)"
+rm -rf "$TMP"
+
+# ── Test 4: Idempotency — running twice doesn't fail or accumulate ──────────
+echo "=== Test 4: idempotency (run twice, no error, same single backup) ==="
+TMP=$(mktemp -d)
+mkdir -p "$TMP/updates"
+echo '{"action":"update","version":"1.14.10","fromVersion":"1.14.8"}' > "$TMP/trigger.json.processing"
+
+bash -c "
+  set -euo pipefail
+  TRIGGER_FILE='$TMP/trigger.json'
+  RESULT_FILE='$TMP/result.json'
+  TRIGGER_PROCESSING=\"\${TRIGGER_FILE}.processing\"
+  UPDATE_STATE_BACKUP_PREFIX='update-state'
+  log() { :; }
+  $EXTRACTED
+  backup_update_state '$TMP/backup/v1.14.8'
+  backup_update_state '$TMP/backup/v1.14.8'
+" 2>&1 | grep -E "(ERROR|FAIL)" || true
+
+assert_eq "only one trigger backup after 2 runs" "1" "$(find "$TMP/backup/v1.14.8/update-state" -name "trigger.json.processing" 2>/dev/null | wc -l)"
+rm -rf "$TMP"
+
+# ── Test 5: main() calls backup_update_state in the right spot ──────────────
+echo "=== Test 5: main() flow — backup_update_state called after backup_data_dir ==="
+if grep -q 'backup_update_state "\$backup_dir"' "$UPDATE_SCRIPT"; then
+  echo "  PASS: backup_update_state invoked from main()"
+  pass=$(( pass + 1 ))
+else
+  echo "  FAIL: backup_update_state not invoked from main()"
+  fail=$(( fail + 1 ))
+fi
+
+# Verify ordering: backup_update_state must come AFTER backup_data_dir in main()
+DATA_LINE=$(grep -n 'backup_data_dir "\$backup_dir"' "$UPDATE_SCRIPT" | head -1 | cut -d: -f1)
+STATE_LINE=$(grep -n 'backup_update_state "\$backup_dir"' "$UPDATE_SCRIPT" | head -1 | cut -d: -f1)
+if (( STATE_LINE > DATA_LINE )); then
+  echo "  PASS: backup_update_state (line $STATE_LINE) called AFTER backup_data_dir (line $DATA_LINE)"
+  pass=$(( pass + 1 ))
+else
+  echo "  FAIL: backup_update_state (line $STATE_LINE) is NOT after backup_data_dir (line $DATA_LINE)"
+  fail=$(( fail + 1 ))
+fi
+
+# ── Summary ──────────────────────────────────────────────────────────────────
+echo
+echo "═══════════════════════════════════════════"
+echo "  backup_update_state() regression test"
+echo "  PASS: $pass    FAIL: $fail"
+echo "═══════════════════════════════════════════"
+
+if (( fail > 0 )); then
+  exit 1
+fi
+echo "All tests passed."

scripts/dashcaddy-update.sh

@@ -26,6 +26,18 @@ readonly CHANNEL_CONF="${UPDATES_DIR}/channel.conf"
 readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data"
 readonly DATA_BACKUP_PREFIX="data-backup"
 
+# Updater state (trigger.json / result.json) backup — keeps the audit trail
+# (what version we were attempting, what the previous update's outcome was) tied
+# to the same versioned backup directory as code + data. After a failed update,
+# operators can inspect what was attempted without correlating timestamps, and
+# rollback tooling can reconstruct a "what just happened" view of the update
+# state machine. NOTE: we do NOT auto-restore trigger.json on rollback — the
+# rollback handler reads a fresh trigger.json written by the operator/container;
+# restoring the previous attempt's trigger would clobber the active rollback
+# request. Backups here are read-only forensic evidence.
+readonly UPDATE_STATE_BACKUP_PREFIX="update-state"
+readonly TRIGGER_PROCESSING="${TRIGGER_FILE}.processing"
+
 log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
 
 # Decide if a given release channel is acceptable on this host.
@@ -112,6 +124,40 @@ backup_data_dir() {
   fi
 }
 
+# ── Updater state backup (trigger.json.processing + result.json) ─────────────
+# Captures what was being attempted + the last result so post-mortem can answer
+# "why did this fail" without joining timestamps across files. Tolerates absent
+# files (first-ever run) and locked files (chattr +i). Idempotent — re-running
+# overwrites the previous backup.
+backup_update_state() {
+  local backup_dir="$1"
+  local state_dir="${backup_dir}/${UPDATE_STATE_BACKUP_PREFIX}"
+  mkdir -p "$state_dir"
+
+  local copied=0
+  for src in "$TRIGGER_PROCESSING" "$RESULT_FILE"; do
+    if [[ -f "$src" ]]; then
+      # Unlock temporarily if immutable, copy, re-lock.
+      local was_locked=false
+      if lsattr -d "$src" 2>/dev/null | awk '{exit !($1 ~ /i/)}'; then
+        was_locked=true
+        chattr -i "$src" 2>/dev/null || true
+      fi
+      cp -f "$src" "${state_dir}/$(basename "$src")" 2>/dev/null && copied=$(( copied + 1 ))
+      if [[ "$was_locked" == "true" ]]; then
+        chattr +i "$src" 2>/dev/null || true
+      fi
+    fi
+  done
+
+  if (( copied > 0 )); then
+    log "Update-state backup: ${copied} file(s) -> ${state_dir}"
+  else
+    log "Update-state backup: nothing to back up (no trigger/result files)"
+    rmdir "$state_dir" 2>/dev/null || true
+  fi
+}
+
 # ── Data restore ──────────────────────────────────────────────────────────────
 restore_data_dir() {
   local backup_dir="$1"
@@ -329,6 +375,10 @@ main() {
   # Backup data/ directory (services.json, config.json, credentials, etc.)
   backup_data_dir "$backup_dir"
 
+  # Backup updater state (trigger.json.processing + result.json) so post-mortem
+  # has a forensic trail tied to this exact version's backup.
+  backup_update_state "$backup_dir"
+
   cleanup_old_backups
 
   # 3. Copy new files from staging to API source