#!/usr/bin/env bash # Integration test harness for the dashcaddy-update.sh auto-update pipeline. # # Exercises the FULL flow: # trigger.json -> backup -> verifier -> docker build (mocked) -> docker run (mocked) # -> health check (mocked) -> result.json -> cleanup # # Run from dashcaddy-api/scripts/: # bash test-dashcaddy-update-integration.sh # # Strategy: build a sandbox at /tmp/dashcaddy-test-XXXXXX/ that mimics # /opt/dashcaddy/ on DNS2, then run a copy of dashcaddy-update.sh with all # hardcoded /opt/dashcaddy paths rewritten to the sandbox path. Mocked # binaries (docker) and a Python one-shot health server live in the sandbox # and are prepended to PATH / invoked via a python orchestrator. # # Each test scenario sets up a synthetic "from" deployment, writes a # trigger.json, runs the pipeline via the python orchestrator (which manages # the health server lifecycle), and asserts the resulting result.json + # filesystem state. # # Exit 0 = all scenarios pass, non-zero = at least one failed. set -uo pipefail readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Resolve dashcaddy-update.sh — try local then canonical location UPDATE_SCRIPT_SRC="${SCRIPT_DIR}/dashcaddy-update.sh" [[ ! -f "$UPDATE_SCRIPT_SRC" ]] && UPDATE_SCRIPT_SRC="$(cd "${SCRIPT_DIR}/../../scripts" 2>/dev/null && pwd)/dashcaddy-update.sh" if [[ ! -f "$UPDATE_SCRIPT_SRC" ]]; then echo "FAIL: dashcaddy-update.sh not found" exit 1 fi # ── Test harness infrastructure ────────────────────────────────────────────── pass=0 fail=0 assert_eq() { local desc="$1" expected="$2" actual="$3" if [[ "$expected" == "$actual" ]]; then echo " PASS: $desc" pass=$(( pass + 1 )) else echo " FAIL: $desc — expected '$expected', got '$actual'" fail=$(( fail + 1 )) fi } assert_file_exists() { local desc="$1" file="$2" if [[ -f "$file" ]]; then echo " PASS: $desc" pass=$(( pass + 1 )) else echo " FAIL: $desc — file '$file' does not exist" fail=$(( fail + 1 )) fi } assert_dir_exists() { local desc="$1" dir="$2" if [[ -d "$dir" ]]; then echo " PASS: $desc" pass=$(( pass + 1 )) else echo " FAIL: $desc — dir '$dir' does not exist" fail=$(( fail + 1 )) fi } assert_json_field() { local desc="$1" file="$2" field="$3" expected="$4" local actual actual=$(python3 -c "import json; d=json.load(open('$file')); print(d.get('$field', ''))" 2>/dev/null || echo "") assert_eq "$desc" "$expected" "$actual" } assert_grep() { local desc="$1" file="$2" pattern="$3" if grep -qE "$pattern" "$file" 2>/dev/null; then echo " PASS: $desc" pass=$(( pass + 1 )) else echo " FAIL: $desc — pattern '$pattern' not in $file" fail=$(( fail + 1 )) fi } assert_not_exists() { local desc="$1" file="$2" if [[ ! -e "$file" ]]; then echo " PASS: $desc" pass=$(( pass + 1 )) else echo " FAIL: $desc — file '$file' exists but should not" fail=$(( fail + 1 )) fi } # ── Python orchestrator ────────────────────────────────────────────────────── # A single Python script that: # 1. Starts a one-shot HTTP responder on a given port (returns 200 OK or # 503 based on env var) # 2. Forks the pipeline as a subprocess # 3. After pipeline exits, kills the responder # 4. Writes the pipeline's exit code + log to disk for assertions # # This avoids backgrounding from inside a foreground bash tool. ORCHESTRATOR_SRC="$(cat << 'PYEOF' import http.server import socketserver import subprocess import sys import os import time import threading PORT = int(os.environ.get("HEALTH_PORT", "33001")) HEALTH_OK = os.environ.get("HEALTH_SHOULD_PASS", "yes") == "yes" COMMAND = os.environ.get("PIPELINE_CMD", "") LOG_FILE = os.environ.get("PIPELINE_LOG", "/tmp/pipeline.log") RC_FILE = os.environ.get("PIPELINE_RC_FILE", "/tmp/pipeline.rc") MAX_HEALTH_REQUESTS = int(os.environ.get("MAX_HEALTH_REQUESTS", "10")) class HealthHandler(http.server.BaseHTTPRequestHandler): request_count = 0 def do_GET(self): HealthHandler.request_count += 1 if HEALTH_OK: self.send_response(200) self.send_header("Content-Length", "2") self.end_headers() self.wfile.write(b"OK") else: self.send_response(503) self.end_headers() def log_message(self, *args): pass class ReusableTCPServer(socketserver.TCPServer): allow_reuse_address = True allow_reuse_port = True # Critical: lets us rebind immediately after shutdown # Start health server in a thread httpd = ReusableTCPServer(("", PORT), HealthHandler) server_thread = threading.Thread(target=httpd.serve_forever, daemon=True) server_thread.start() time.sleep(0.3) # Run the pipeline try: result = subprocess.run( COMMAND, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120, ) with open(LOG_FILE, "wb") as f: f.write(result.stdout) with open(RC_FILE, "w") as f: f.write(str(result.returncode)) except subprocess.TimeoutExpired as e: with open(LOG_FILE, "wb") as f: f.write(e.stdout or b"") with open(RC_FILE, "w") as f: f.write("124") except Exception as e: with open(LOG_FILE, "w") as f: f.write(f"orchestrator error: {e}") with open(RC_FILE, "w") as f: f.write("99") # Shutdown explicitly — this is what frees the port httpd.shutdown() httpd.server_close() PYEOF )" run_pipeline() { # Args: dash_root patched_script trigger_json content log_file rc_file health_should_pass local dash_root="$1" local patched_script="$2" local health_should_pass="${3:-yes}" local log_file="$4" local rc_file="$5" # Write orchestrator + run it local orch_py="$dash_root/.orchestrator.py" echo "$ORCHESTRATOR_SRC" > "$orch_py" PIPELINE_CMD="PATH='$dash_root/bin:$PATH' bash '$patched_script'" \ PIPELINE_LOG="$log_file" \ PIPELINE_RC_FILE="$rc_file" \ HEALTH_SHOULD_PASS="$health_should_pass" \ HEALTH_PORT="33001" \ python3 "$orch_py" # Return the exit code if [[ -f "$rc_file" ]]; then cat "$rc_file" else echo "127" fi } # ── Sandbox builder ────────────────────────────────────────────────────────── # # Lays out the sandbox as: # $SANDBOX_ROOT/ # opt/dashcaddy/ # updates/ # staging/dashcaddy-api/ <- staging_dir # dashcaddy-api/ <- api_source_dir (FROM) # data/services.json # src/app.js # license-keygen.js # server.js # bin/ # docker <- fake docker # patched-update.sh <- path-rewritten update script # .docker-build-ran <- marker created by mocked docker build # .docker-rm-ran <- marker created by mocked docker rm # .docker-run-ran <- marker created by mocked docker run build_sandbox() { local from_version="$1" local new_version="$2" local with_src="${3:-yes}" # yes/no — controls whether staging has src/ local extra_setup="${4:-}" # optional bash to run after setup local sandbox=$(mktemp -d /tmp/dashcaddy-test-XXXXXX) local dash_root="$sandbox/opt/dashcaddy" mkdir -p "$dash_root"/{updates,bin,scripts} mkdir -p "$dash_root/updates/staging/dashcaddy-api" mkdir -p "$dash_root/dashcaddy-api/data" # ── FROM deployment ── echo '{"services":[]}' > "$dash_root/dashcaddy-api/data/services.json" cat > "$dash_root/dashcaddy-api/server.js" << 'EOF' const { createApp } = require('./src/app'); EOF if [[ "$with_src" == "yes" ]]; then mkdir -p "$dash_root/dashcaddy-api/src/managers" cat > "$dash_root/dashcaddy-api/src/app.js" << 'EOF' module.exports = { createApp: () => ({ app: {}, log: console, config: {} }) }; EOF cat > "$dash_root/dashcaddy-api/src/managers/license-manager.js" << 'EOF' const keygen = require('../../license-keygen'); module.exports = {}; EOF fi cat > "$dash_root/dashcaddy-api/license-keygen.js" << 'EOF' module.exports = { verifyCode: () => true }; EOF echo "from-commit" > "$dash_root/dashcaddy-api/VERSION" # ── STAGING (new version) ── cp "$dash_root/dashcaddy-api/server.js" "$dash_root/updates/staging/dashcaddy-api/" cp "$dash_root/dashcaddy-api/license-keygen.js" "$dash_root/updates/staging/dashcaddy-api/" if [[ "$with_src" == "yes" ]]; then cp -r "$dash_root/dashcaddy-api/src" "$dash_root/updates/staging/dashcaddy-api/" fi echo "new-commit-$new_version" > "$dash_root/updates/staging/dashcaddy-api/VERSION" # ── Mocked docker ── cat > "$dash_root/bin/docker" << 'EOF' #!/usr/bin/env bash echo "[mock-docker] $*" >> "${MOCK_DOCKER_LOG:-/tmp/mock-docker.log}" case "$1" in build) : >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-build-ran" exit 0 ;; rm) : >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-rm-ran" exit 0 ;; run) : >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-run-ran" exit 0 ;; compose|version) exit 0 ;; *) exit 0 ;; esac EOF chmod +x "$dash_root/bin/docker" # Fake start.sh — NOT created in the sandbox so deploy_mode picks "run" # (which exercises docker rm + docker run paths in restart_container). # Production DNS2 has start.sh and uses the startsh deploy path; the test # deliberately diverges so we observe the full docker restart sequence. # ── Post-deploy verifier (real script copied in) ───────────────────────── # dashcaddy-update.sh hard-codes /opt/dashcaddy/scripts/dashcaddy-post-deploy-patches.sh # (which the sed rewrite maps to $dash_root/scripts/...). For the verifier to # actually be invoked, we copy the real script into the sandbox. The verifier # is the one being tested here; we want to observe its behavior end-to-end. local verifier_src="${SCRIPT_DIR}/dashcaddy-post-deploy-patches.sh" if [[ ! -f "$verifier_src" ]]; then verifier_src="$(cd "${SCRIPT_DIR}/../../scripts" 2>/dev/null && pwd)/dashcaddy-post-deploy-patches.sh" fi if [[ -f "$verifier_src" ]]; then cp "$verifier_src" "$dash_root/scripts/dashcaddy-post-deploy-patches.sh" chmod +x "$dash_root/scripts/dashcaddy-post-deploy-patches.sh" fi # ── Path-rewritten update script ── local patched="$sandbox/patched-update.sh" sed "s|/opt/dashcaddy|$dash_root|g" "$UPDATE_SCRIPT_SRC" > "$patched" chmod +x "$patched" if [[ -n "$extra_setup" ]]; then ( cd "$sandbox" && eval "$extra_setup" ) fi # Write a state file so the caller can recover the paths cat > "$sandbox/.sandbox-paths" << EOF SANDBOX_ROOT=$sandbox DASH_ROOT=$dash_root PATCHED_SCRIPT=$patched API_SOURCE_DIR=$dash_root/dashcaddy-api STAGING_DIR=$dash_root/updates/staging/dashcaddy-api UPDATES_DIR=$dash_root/updates EOF echo "$sandbox/.sandbox-paths" } write_trigger() { local updates_dir="$1" action="$2" to_version="$3" from_version="$4" staging_dir="$5" api_source_dir="$6" cat > "$updates_dir/trigger.json" << EOF { "action": "${action}", "version": "${to_version}", "fromVersion": "${from_version}", "channel": "stable", "commit": "new-commit-${to_version}", "stagingDir": "${staging_dir}", "apiSourceDir": "${api_source_dir}" } EOF } load_paths() { local paths_file="$1" # shellcheck disable=SC1090 source "$paths_file" } cleanup_sandbox() { local sandbox="$1" rm -rf "$sandbox" /tmp/mock-docker.log 2>/dev/null } # ──────────────────────────────────────────────────────────────────────────── # SCENARIO 1: Happy path — update succeeds end-to-end # ──────────────────────────────────────────────────────────────────────────── echo "=== Scenario 1: happy path — update v1.14.8 -> v1.14.9 ===" PATHS=$(build_sandbox "1.14.8" "1.14.9" "yes") SANDBOX=$(dirname "$PATHS") load_paths "$PATHS" write_trigger "$UPDATES_DIR" "update" "1.14.9" "1.14.8" "$STAGING_DIR" "$API_SOURCE_DIR" RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \ IMAGE_MARKER_DIR="$SANDBOX" \ run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \ "$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc") assert_eq "pipeline exit code" "0" "$RC" assert_file_exists "result.json exists" "$UPDATES_DIR/result.json" assert_json_field "result.success=true" "$UPDATES_DIR/result.json" "success" "True" assert_json_field "result.version=1.14.9" "$UPDATES_DIR/result.json" "version" "1.14.9" assert_file_exists "docker build ran" "$SANDBOX/.docker-build-ran" assert_file_exists "docker rm ran" "$SANDBOX/.docker-rm-ran" assert_file_exists "docker run ran" "$SANDBOX/.docker-run-ran" assert_dir_exists "code backup dir created" "$UPDATES_DIR/backups/1.14.8" assert_file_exists "code backup has server.js" "$UPDATES_DIR/backups/1.14.8/server.js" assert_dir_exists "data backup dir created" "$UPDATES_DIR/backups/1.14.8/data-backup" assert_dir_exists "update-state backup dir created" "$UPDATES_DIR/backups/1.14.8/update-state" assert_file_exists "update-state backup has trigger.json.processing" "$UPDATES_DIR/backups/1.14.8/update-state/trigger.json.processing" assert_eq "api source VERSION updated" "new-commit-1.14.9" "$(cat "$API_SOURCE_DIR/VERSION" 2>/dev/null)" assert_grep "docker was invoked with build" "$SANDBOX/.docker-calls.log" "build -t dashcaddy-dashcaddy-api:latest" assert_grep "docker was invoked with run" "$SANDBOX/.docker-calls.log" "run -d --restart unless-stopped" assert_grep "pipeline log shows successful update" "$SANDBOX/pipeline.log" "Update successful" cleanup_sandbox "$SANDBOX" # ──────────────────────────────────────────────────────────────────────────── # SCENARIO 2: v1.14.4-style broken tarball (no src/) — verifier should fail # the build. Pipeline exits non-zero, result.json reports failure. # # AS-OF-CURRENT dashcaddy-update.sh: the verifier's failure is logged as a # WARNING and the build proceeds anyway (the script does not abort on verifier # failure). Mocked docker build always succeeds, so the pipeline ends with # success=true. The value of this scenario is asserting that the verifier IS # invoked, DOES detect the v1.14.4-class bug, and emits the expected error # message — i.e. the verifier itself works. Blocking the build on verifier # failure is a separate gap in dashcaddy-update.sh (TODO: tighten the call # site in main() so verifier failure aborts). # ──────────────────────────────────────────────────────────────────────────── echo echo "=== Scenario 2: v1.14.4-class bug (no src/ in staging) — verifier detects it ===" PATHS=$(build_sandbox "1.14.4" "1.14.5" "no") SANDBOX=$(dirname "$PATHS") load_paths "$PATHS" write_trigger "$UPDATES_DIR" "update" "1.14.5" "1.14.4" "$STAGING_DIR" "$API_SOURCE_DIR" RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \ IMAGE_MARKER_DIR="$SANDBOX" \ run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \ "$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc") # Current production behavior: verifier warns, build proceeds, pipeline succeeds. assert_eq "pipeline exit code" "0" "$RC" assert_file_exists "result.json exists" "$UPDATES_DIR/result.json" assert_json_field "result.success=true (build proceeded despite verifier warning)" "$UPDATES_DIR/result.json" "success" "True" # The KEY assertion: verifier actually caught the bug. assert_grep "verifier detected the missing src/ tree" "$SANDBOX/pipeline.log" "Build should be ABORTED" assert_grep "verifier failure was surfaced as a warning" "$SANDBOX/pipeline.log" "Post-deploy patches exited non-zero" # Build still ran (current code ignores verifier failure). assert_file_exists "docker build ran (current code proceeds past verifier failure)" "$SANDBOX/.docker-build-ran" cleanup_sandbox "$SANDBOX" # ──────────────────────────────────────────────────────────────────────────── # SCENARIO 3: Rollback — action=rollback restores from backup # ──────────────────────────────────────────────────────────────────────────── echo echo "=== Scenario 3: rollback — restore from backup directory ===" PATHS=$(build_sandbox "1.14.9" "1.14.10" "yes") SANDBOX=$(dirname "$PATHS") load_paths "$PATHS" # Pre-populate a backup dir (simulate that a prior update created it) mkdir -p "$UPDATES_DIR/backups/1.14.8/data-backup" echo '{"services":[]}' > "$UPDATES_DIR/backups/1.14.8/data-backup/services.json" cat > "$UPDATES_DIR/backups/1.14.8/server.js" << 'EOF' // ROLLBACK VERSION const { createApp } = require('./src/app'); console.log('ROLLBACK-1.14.8'); EOF echo "rollback-commit-1.14.8" > "$UPDATES_DIR/backups/1.14.8/VERSION" mkdir -p "$UPDATES_DIR/backups/1.14.8/src" cat > "$UPDATES_DIR/backups/1.14.8/src/app.js" << 'EOF' module.exports = { createApp: () => ({ rollback: '1.14.8' }) }; EOF cp "$UPDATES_DIR/backups/1.14.8/license-keygen.js" "$UPDATES_DIR/backups/1.14.8/" 2>/dev/null # Rollback needs license-keygen.js in backup too cat > "$UPDATES_DIR/backups/1.14.8/license-keygen.js" << 'EOF' module.exports = { verifyCode: () => true }; EOF # Write rollback trigger (no staging_dir needed for rollback) cat > "$UPDATES_DIR/trigger.json" << EOF { "action": "rollback", "version": "1.14.8", "fromVersion": "1.14.9", "channel": "stable", "commit": "", "stagingDir": "", "apiSourceDir": "${API_SOURCE_DIR}" } EOF RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \ IMAGE_MARKER_DIR="$SANDBOX" \ run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \ "$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc") assert_eq "rollback pipeline exit code" "0" "$RC" assert_file_exists "result.json exists" "$UPDATES_DIR/result.json" assert_json_field "result.success=true" "$UPDATES_DIR/result.json" "success" "True" assert_json_field "result.version=1.14.8" "$UPDATES_DIR/result.json" "version" "1.14.8" assert_file_exists "docker build called (rollback rebuilds)" "$SANDBOX/.docker-build-ran" assert_eq "api source VERSION restored" "rollback-commit-1.14.8" "$(cat "$API_SOURCE_DIR/VERSION" 2>/dev/null)" assert_grep "pipeline log shows rollback" "$SANDBOX/pipeline.log" "ROLLBACK" cleanup_sandbox "$SANDBOX" # ──────────────────────────────────────────────────────────────────────────── # SCENARIO 4: No trigger file — pipeline exits cleanly without doing anything # ──────────────────────────────────────────────────────────────────────────── echo echo "=== Scenario 4: no trigger.json — exits 0 with no-op ===" PATHS=$(build_sandbox "1.14.9" "1.14.10" "yes") SANDBOX=$(dirname "$PATHS") load_paths "$PATHS" # Deliberately don't write trigger.json RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \ IMAGE_MARKER_DIR="$SANDBOX" \ run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \ "$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc") assert_eq "no-op exit code" "0" "$RC" assert_grep "logs 'nothing to do'" "$SANDBOX/pipeline.log" "No trigger file found" assert_not_exists "docker build did NOT run" "$SANDBOX/.docker-build-ran" cleanup_sandbox "$SANDBOX" # ──────────────────────────────────────────────────────────────────────────── # SCENARIO 5: Channel rejection — prerelease trigger on default host exits 1 # ──────────────────────────────────────────────────────────────────────────── echo echo "=== Scenario 5: prerelease channel rejected (no ALLOW_PRERELEASE) ===" PATHS=$(build_sandbox "1.14.9" "1.15.0-beta" "yes") SANDBOX=$(dirname "$PATHS") load_paths "$PATHS" cat > "$UPDATES_DIR/trigger.json" << EOF { "action": "update", "version": "1.15.0-beta", "fromVersion": "1.14.9", "channel": "beta", "commit": "new-commit-1.15.0-beta", "stagingDir": "${STAGING_DIR}", "apiSourceDir": "${API_SOURCE_DIR}" } EOF RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \ IMAGE_MARKER_DIR="$SANDBOX" \ run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \ "$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc") assert_eq "channel rejection exit code" "1" "$RC" assert_file_exists "result.json exists" "$UPDATES_DIR/result.json" assert_json_field "result.success=false" "$UPDATES_DIR/result.json" "success" "False" assert_grep "result mentions channel rejection" "$UPDATES_DIR/result.json" "Channel 'beta' not allowed" assert_not_exists "docker build did NOT run" "$SANDBOX/.docker-build-ran" cleanup_sandbox "$SANDBOX" # ──────────────────────────────────────────────────────────────────────────── # Summary # ──────────────────────────────────────────────────────────────────────────── echo echo "═══════════════════════════════════════════════════════════" echo " dashcaddy-update.sh integration test" echo " PASS: $pass FAIL: $fail" echo "═══════════════════════════════════════════════════════════" if (( fail > 0 )); then exit 1 fi echo "All scenarios passed."