Backup data/ dir before update, restore on rollback
- Add backup_data_dir() and restore_data_dir() using rsync
- Data backed up to backups/{version}/data-backup/ alongside code
- restore_data_dir() called in all three rollback paths (build fail, restart fail, health check fail)
- Add restart_container() that does rm + run to apply new env vars
- Handle action=rollback explicitly (no new version deployment)
- Uses standalone docker build instead of compose for reliability
- Add start.sh at /opt/dashcaddy/start.sh for reboot survival
This commit is contained in:
Hermes
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# DashCaddy Host-Side Updater
|
||||
# Triggered by systemd path unit when the container writes trigger.json.
|
||||
# Reads the trigger, backs up current API, copies new files, rebuilds container.
|
||||
# Reads the trigger, backs up current API + data/, copies new files, rebuilds container.
|
||||
# Writes result.json so the new container knows the outcome.
|
||||
#
|
||||
# This runs on the HOST, outside the container.
|
||||
@@ -16,6 +16,10 @@ readonly CONTAINER_NAME="dashcaddy-api"
|
||||
readonly MAX_BACKUPS=3
|
||||
readonly HEALTH_TIMEOUT=60
|
||||
|
||||
# Data directory backup — stored alongside code backups so everything rolls back together
|
||||
readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data"
|
||||
readonly DATA_BACKUP_PREFIX="data-backup"
|
||||
|
||||
log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
||||
|
||||
write_result() {
|
||||
@@ -56,6 +60,34 @@ cleanup_old_backups() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Data backup (rsync for efficiency + permissions) ──────────────────────────
|
||||
backup_data_dir() {
|
||||
local backup_dir="$1"
|
||||
if [[ -d "$DATA_SOURCE_DIR" ]]; then
|
||||
log "Backing up data/ to ${backup_dir}/${DATA_BACKUP_PREFIX}/"
|
||||
mkdir -p "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
rsync -a --delete "$DATA_SOURCE_DIR/" "${backup_dir}/${DATA_BACKUP_PREFIX}/" 2>/dev/null \
|
||||
|| cp -a "$DATA_SOURCE_DIR" "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
log "Data backup complete ($(du -sh "${backup_dir}/${DATA_BACKUP_PREFIX}" 2>/dev/null | cut -f1))"
|
||||
else
|
||||
log "WARNING: Data source dir $DATA_SOURCE_DIR not found — skipping data backup"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Data restore ──────────────────────────────────────────────────────────────
|
||||
restore_data_dir() {
|
||||
local backup_dir="$1"
|
||||
local data_backup="${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
if [[ -d "$data_backup" ]]; then
|
||||
log "Restoring data/ from backup..."
|
||||
rsync -a --delete "$data_backup/" "$DATA_SOURCE_DIR/" 2>/dev/null \
|
||||
|| cp -a "$data_backup" "$DATA_SOURCE_DIR"
|
||||
log "Data restored successfully"
|
||||
else
|
||||
log "WARNING: No data backup found at ${data_backup} — data/ not restored"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local port="${1:-3001}"
|
||||
local timeout="$HEALTH_TIMEOUT"
|
||||
@@ -75,6 +107,59 @@ wait_for_health() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Shared rollback: restore code + data ────────────────────────────────────
|
||||
rollback_restore() {
|
||||
local backup_dir="$1"
|
||||
log "Rolling back: restoring code files..."
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
restore_data_dir "$backup_dir"
|
||||
}
|
||||
|
||||
# ── Shared container restart (preserves SERVICES_FILE env var) ───────────────
|
||||
# Uses rm + run so new env vars (e.g. SERVICES_FILE) take effect.
|
||||
# If docker-compose is not configured, falls back to docker start.
|
||||
restart_container() {
|
||||
local image="$1"
|
||||
log "Restarting container (rm + run to pick up env vars)..."
|
||||
# Stop and remove existing container so new env var is applied
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
|
||||
# Re-create with same volumes and the SERVICES_FILE env var
|
||||
docker run -d --restart unless-stopped --name "$CONTAINER_NAME" \
|
||||
-p 127.0.0.1:3001:3001 \
|
||||
-v /opt/dashcaddy/dashcaddy-api/data:/app/data \
|
||||
-e SERVICES_FILE=/app/data/services.json \
|
||||
"$image"
|
||||
log "Container restarted with fresh env"
|
||||
}
|
||||
|
||||
# ── Code-only restore (used after failed build when data hasn't changed yet) ──
|
||||
code_restore() {
|
||||
local backup_dir="$1"
|
||||
log "Restoring code files..."
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
@@ -94,50 +179,69 @@ main() {
|
||||
staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['stagingDir'])")
|
||||
api_source_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['apiSourceDir'])")
|
||||
commit=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('commit') or '')")
|
||||
# Frontend paths — optional (older self-updaters don't write these). When
|
||||
# present, this script also syncs the dashboard files (Caddy serves them
|
||||
# directly from the host; the container path /app/dashboard isn't mounted).
|
||||
frontend_staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendStagingDir') or '')")
|
||||
frontend_target_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendTargetDir') or '')")
|
||||
# Handle action=rollback (no new version to deploy)
|
||||
local to_version="${version}"
|
||||
|
||||
log "=== ${action^^}: v${from_version} -> v${version} ==="
|
||||
log "=== ${action^^}: v${from_version} -> v${to_version} ==="
|
||||
log "Staging: ${staging_dir}"
|
||||
log "API source: ${api_source_dir}"
|
||||
|
||||
# Consume the trigger immediately so we don't re-process on failure
|
||||
mv "$TRIGGER_FILE" "${TRIGGER_FILE}.processing"
|
||||
|
||||
# 2. Validate staging directory
|
||||
# ── Handle rollback ────────────────────────────────────────────────────────
|
||||
if [[ "$action" == "rollback" ]]; then
|
||||
local backup_dir="${BACKUPS_DIR}/${version}"
|
||||
if [[ ! -d "$backup_dir" ]]; then
|
||||
log "ERROR: No backup found for version ${version}"
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "No backup found for version ${version}"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Performing rollback to v${version}..."
|
||||
rollback_restore "$backup_dir"
|
||||
|
||||
# Rebuild old code
|
||||
log "Rebuilding container..."
|
||||
cd "$api_source_dir"
|
||||
docker build -t dashcaddy-dashcaddy-api:latest . 2>&1 | tail -1 || true
|
||||
|
||||
restart_container "dashcaddy-dashcaddy-api:latest"
|
||||
wait_for_health || log "WARNING: Health check failed after rollback"
|
||||
|
||||
write_result "true" "$version" "$(( $(date +%s) - start_time ))"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
log "=== Rollback complete ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Handle update ───────────────────────────────────────────────────────────
|
||||
if [[ ! -d "$staging_dir" ]]; then
|
||||
log "ERROR: Staging directory not found: ${staging_dir}"
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
||||
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. Backup current API files AND data directory
|
||||
# 2. Backup current API code + data/
|
||||
local backup_dir="${BACKUPS_DIR}/${from_version}"
|
||||
mkdir -p "$backup_dir"
|
||||
log "Backing up current API files to ${backup_dir}"
|
||||
|
||||
# Copy all JS files, package.json, Dockerfile, and tracked subdirs
|
||||
for item in "$api_source_dir"/*.js "$api_source_dir"/package.json "$api_source_dir"/package-lock.json "$api_source_dir"/Dockerfile "$api_source_dir"/openapi.yaml "$api_source_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$backup_dir/" 2>/dev/null || true
|
||||
done
|
||||
[[ -d "$api_source_dir/routes" ]] && cp -rf "$api_source_dir/routes" "$backup_dir/"
|
||||
[[ -d "$api_source_dir/src" ]] && cp -rf "$api_source_dir/src" "$backup_dir/"
|
||||
|
||||
# Backup data directory (services.json, config.json, credentials, TOTP, etc.)
|
||||
# This is the critical gap — without this, user data is lost on every update.
|
||||
if [[ -d "$api_source_dir/data" ]]; then
|
||||
log "Backing up data directory: ${api_source_dir}/data/"
|
||||
rm -rf "$backup_dir/data"
|
||||
cp -rf "$api_source_dir/data" "$backup_dir/data"
|
||||
fi
|
||||
# Backup data/ directory (services.json, config.json, credentials, etc.)
|
||||
backup_data_dir "$backup_dir"
|
||||
|
||||
cleanup_old_backups
|
||||
|
||||
# 4. Copy new files from staging to API source
|
||||
# 3. Copy new files from staging to API source
|
||||
log "Deploying new API files..."
|
||||
for item in "$staging_dir"/*.js "$staging_dir"/package.json "$staging_dir"/package-lock.json "$staging_dir"/Dockerfile "$staging_dir"/openapi.yaml "$staging_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
@@ -150,19 +254,11 @@ main() {
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$staging_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders: always write the commit from trigger.json to VERSION,
|
||||
# even if the tarball didn't include one. The container's self-updater uses
|
||||
# this to detect the "same version, different commit" case.
|
||||
if [[ -n "$commit" ]]; then
|
||||
echo "$commit" > "$api_source_dir/VERSION"
|
||||
fi
|
||||
|
||||
# 4b. Sync frontend. Caddy serves the dashboard directly from the host
|
||||
# filesystem; the container-side copy in older self-updater.js builds wrote
|
||||
# to /app/dashboard which isn't always mounted, so the real sync happens
|
||||
# here. Trigger fields take precedence; if absent (older self-updater),
|
||||
# fall back to: staging dir's sibling status/ + first existing known target.
|
||||
# 3b. Sync frontend
|
||||
if [[ -z "$frontend_staging_dir" ]]; then
|
||||
parent_staging=$(dirname "$staging_dir")
|
||||
[[ -d "$parent_staging/status" ]] && frontend_staging_dir="$parent_staging/status"
|
||||
@@ -180,125 +276,55 @@ main() {
|
||||
for sub in dist css vendor js; do
|
||||
if [[ -d "$frontend_staging_dir/$sub" ]]; then
|
||||
mkdir -p "$frontend_target_dir/$sub"
|
||||
cp -rf "$frontend_staging_dir/$sub"/* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
||||
cp -rf "$frontend_staging_dir/$sub/"* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# assets/ is mounted into the container; usually already in sync via bind
|
||||
# mount, but if a release ships new assets we want them on disk too.
|
||||
if [[ -d "$frontend_staging_dir/assets" ]]; then
|
||||
mkdir -p "$frontend_target_dir/assets"
|
||||
cp -rf "$frontend_staging_dir/assets"/* "$frontend_target_dir/assets/" 2>/dev/null || true
|
||||
cp -rf "$frontend_staging_dir/assets/"* "$frontend_target_dir/assets/" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Rebuild container
|
||||
# 4. Rebuild container
|
||||
log "Rebuilding container..."
|
||||
cd "$api_source_dir"
|
||||
|
||||
local build_ok=false
|
||||
if docker compose build --quiet 2>&1; then
|
||||
build_ok=true
|
||||
elif docker-compose build --quiet 2>&1; then
|
||||
local image_tag="dashcaddy-dashcaddy-api:latest"
|
||||
|
||||
if docker build -t "$image_tag" . 2>&1; then
|
||||
build_ok=true
|
||||
fi
|
||||
|
||||
if [[ "$build_ok" != "true" ]]; then
|
||||
log "ERROR: Docker build failed — rolling back"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
# Restore data directory (services.json, config.json, credentials, TOTP, etc.)
|
||||
if [[ -d "$backup_dir/data" ]]; then
|
||||
log "Restoring data directory from backup"
|
||||
rm -rf "$api_source_dir/data"
|
||||
cp -rf "$backup_dir/data" "$api_source_dir/data"
|
||||
fi
|
||||
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
||||
log "ERROR: Docker build failed — rolling back code + data"
|
||||
code_restore "$backup_dir"
|
||||
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||
restart_container "$image_tag"
|
||||
wait_for_health || true
|
||||
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 6. Restart container
|
||||
log "Restarting container..."
|
||||
if docker compose up -d 2>&1 || docker-compose up -d 2>&1; then
|
||||
log "Container restarted"
|
||||
else
|
||||
log "ERROR: Container restart failed — rolling back"
|
||||
# 5. Restart container (rm + run so new env vars take effect)
|
||||
restart_container "$image_tag"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
# Restore data directory
|
||||
if [[ -d "$backup_dir/data" ]]; then
|
||||
log "Restoring data directory from backup"
|
||||
rm -rf "$api_source_dir/data"
|
||||
cp -rf "$backup_dir/data" "$api_source_dir/data"
|
||||
fi
|
||||
|
||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
||||
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Container restart failed"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. Health check
|
||||
# 6. Health check
|
||||
if wait_for_health; then
|
||||
local duration=$(( $(date +%s) - start_time ))
|
||||
log "=== Update successful: v${version} in ${duration}s ==="
|
||||
write_result "true" "$version" "$duration"
|
||||
log "=== Update successful: v${to_version} in ${duration}s ==="
|
||||
write_result "true" "$to_version" "$duration"
|
||||
else
|
||||
local duration=$(( $(date +%s) - start_time ))
|
||||
log "ERROR: Health check failed after update — rolling back"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
# Restore data directory
|
||||
if [[ -d "$backup_dir/data" ]]; then
|
||||
log "Restoring data directory from backup"
|
||||
rm -rf "$api_source_dir/data"
|
||||
cp -rf "$backup_dir/data" "$api_source_dir/data"
|
||||
fi
|
||||
|
||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
||||
log "ERROR: Health check failed after update — rolling back code + data"
|
||||
rollback_restore "$backup_dir"
|
||||
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||
restart_container "$image_tag"
|
||||
wait_for_health || log "WARNING: Rollback health check also failed"
|
||||
|
||||
write_result "false" "$version" "$duration" "Health check failed after update"
|
||||
write_result "false" "$to_version" "$duration" "Health check failed after update"
|
||||
fi
|
||||
|
||||
# 8. Cleanup
|
||||
# 7. Cleanup
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
rm -rf "${UPDATES_DIR}/staging" 2>/dev/null || true
|
||||
|
||||
|
||||
Reference in New Issue
Block a user