Files
dashcaddy/dashcaddy-api/scripts/test-dashcaddy-update-integration.sh

553 lines
22 KiB
Bash
Executable File

#!/usr/bin/env bash
# Integration test harness for the dashcaddy-update.sh auto-update pipeline.
#
# Exercises the FULL flow:
# trigger.json -> backup -> verifier -> docker build (mocked) -> docker run (mocked)
# -> health check (mocked) -> result.json -> cleanup
#
# Run from dashcaddy-api/scripts/:
# bash test-dashcaddy-update-integration.sh
#
# Strategy: build a sandbox at /tmp/dashcaddy-test-XXXXXX/ that mimics
# /opt/dashcaddy/ on DNS2, then run a copy of dashcaddy-update.sh with all
# hardcoded /opt/dashcaddy paths rewritten to the sandbox path. Mocked
# binaries (docker) and a Python one-shot health server live in the sandbox
# and are prepended to PATH / invoked via a python orchestrator.
#
# Each test scenario sets up a synthetic "from" deployment, writes a
# trigger.json, runs the pipeline via the python orchestrator (which manages
# the health server lifecycle), and asserts the resulting result.json +
# filesystem state.
#
# Exit 0 = all scenarios pass, non-zero = at least one failed.
set -uo pipefail
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Resolve dashcaddy-update.sh — try local then canonical location
UPDATE_SCRIPT_SRC="${SCRIPT_DIR}/dashcaddy-update.sh"
[[ ! -f "$UPDATE_SCRIPT_SRC" ]] && UPDATE_SCRIPT_SRC="$(cd "${SCRIPT_DIR}/../../scripts" 2>/dev/null && pwd)/dashcaddy-update.sh"
if [[ ! -f "$UPDATE_SCRIPT_SRC" ]]; then
echo "FAIL: dashcaddy-update.sh not found"
exit 1
fi
# ── Test harness infrastructure ──────────────────────────────────────────────
pass=0
fail=0
assert_eq() {
local desc="$1" expected="$2" actual="$3"
if [[ "$expected" == "$actual" ]]; then
echo " PASS: $desc"
pass=$(( pass + 1 ))
else
echo " FAIL: $desc — expected '$expected', got '$actual'"
fail=$(( fail + 1 ))
fi
}
assert_file_exists() {
local desc="$1" file="$2"
if [[ -f "$file" ]]; then
echo " PASS: $desc"
pass=$(( pass + 1 ))
else
echo " FAIL: $desc — file '$file' does not exist"
fail=$(( fail + 1 ))
fi
}
assert_dir_exists() {
local desc="$1" dir="$2"
if [[ -d "$dir" ]]; then
echo " PASS: $desc"
pass=$(( pass + 1 ))
else
echo " FAIL: $desc — dir '$dir' does not exist"
fail=$(( fail + 1 ))
fi
}
assert_json_field() {
local desc="$1" file="$2" field="$3" expected="$4"
local actual
actual=$(python3 -c "import json; d=json.load(open('$file')); print(d.get('$field', '<MISSING>'))" 2>/dev/null || echo "<PARSE_ERROR>")
assert_eq "$desc" "$expected" "$actual"
}
assert_grep() {
local desc="$1" file="$2" pattern="$3"
if grep -qE "$pattern" "$file" 2>/dev/null; then
echo " PASS: $desc"
pass=$(( pass + 1 ))
else
echo " FAIL: $desc — pattern '$pattern' not in $file"
fail=$(( fail + 1 ))
fi
}
assert_not_exists() {
local desc="$1" file="$2"
if [[ ! -e "$file" ]]; then
echo " PASS: $desc"
pass=$(( pass + 1 ))
else
echo " FAIL: $desc — file '$file' exists but should not"
fail=$(( fail + 1 ))
fi
}
# ── Python orchestrator ──────────────────────────────────────────────────────
# A single Python script that:
# 1. Starts a one-shot HTTP responder on a given port (returns 200 OK or
# 503 based on env var)
# 2. Forks the pipeline as a subprocess
# 3. After pipeline exits, kills the responder
# 4. Writes the pipeline's exit code + log to disk for assertions
#
# This avoids backgrounding from inside a foreground bash tool.
ORCHESTRATOR_SRC="$(cat << 'PYEOF'
import http.server
import socketserver
import subprocess
import sys
import os
import time
import threading
PORT = int(os.environ.get("HEALTH_PORT", "33001"))
HEALTH_OK = os.environ.get("HEALTH_SHOULD_PASS", "yes") == "yes"
COMMAND = os.environ.get("PIPELINE_CMD", "")
LOG_FILE = os.environ.get("PIPELINE_LOG", "/tmp/pipeline.log")
RC_FILE = os.environ.get("PIPELINE_RC_FILE", "/tmp/pipeline.rc")
MAX_HEALTH_REQUESTS = int(os.environ.get("MAX_HEALTH_REQUESTS", "10"))
class HealthHandler(http.server.BaseHTTPRequestHandler):
request_count = 0
def do_GET(self):
HealthHandler.request_count += 1
if HEALTH_OK:
self.send_response(200)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"OK")
else:
self.send_response(503)
self.end_headers()
def log_message(self, *args):
pass
class ReusableTCPServer(socketserver.TCPServer):
allow_reuse_address = True
allow_reuse_port = True # Critical: lets us rebind immediately after shutdown
# Start health server in a thread
httpd = ReusableTCPServer(("", PORT), HealthHandler)
server_thread = threading.Thread(target=httpd.serve_forever, daemon=True)
server_thread.start()
time.sleep(0.3)
# Run the pipeline
try:
result = subprocess.run(
COMMAND,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=120,
)
with open(LOG_FILE, "wb") as f:
f.write(result.stdout)
with open(RC_FILE, "w") as f:
f.write(str(result.returncode))
except subprocess.TimeoutExpired as e:
with open(LOG_FILE, "wb") as f:
f.write(e.stdout or b"")
with open(RC_FILE, "w") as f:
f.write("124")
except Exception as e:
with open(LOG_FILE, "w") as f:
f.write(f"orchestrator error: {e}")
with open(RC_FILE, "w") as f:
f.write("99")
# Shutdown explicitly — this is what frees the port
httpd.shutdown()
httpd.server_close()
PYEOF
)"
run_pipeline() {
# Args: dash_root patched_script trigger_json content log_file rc_file health_should_pass
local dash_root="$1"
local patched_script="$2"
local health_should_pass="${3:-yes}"
local log_file="$4"
local rc_file="$5"
# Write orchestrator + run it
local orch_py="$dash_root/.orchestrator.py"
echo "$ORCHESTRATOR_SRC" > "$orch_py"
PIPELINE_CMD="PATH='$dash_root/bin:$PATH' bash '$patched_script'" \
PIPELINE_LOG="$log_file" \
PIPELINE_RC_FILE="$rc_file" \
HEALTH_SHOULD_PASS="$health_should_pass" \
HEALTH_PORT="33001" \
python3 "$orch_py"
# Return the exit code
if [[ -f "$rc_file" ]]; then
cat "$rc_file"
else
echo "127"
fi
}
# ── Sandbox builder ──────────────────────────────────────────────────────────
#
# Lays out the sandbox as:
# $SANDBOX_ROOT/
# opt/dashcaddy/
# updates/
# staging/dashcaddy-api/ <- staging_dir
# dashcaddy-api/ <- api_source_dir (FROM)
# data/services.json
# src/app.js
# license-keygen.js
# server.js
# bin/
# docker <- fake docker
# patched-update.sh <- path-rewritten update script
# .docker-build-ran <- marker created by mocked docker build
# .docker-rm-ran <- marker created by mocked docker rm
# .docker-run-ran <- marker created by mocked docker run
build_sandbox() {
local from_version="$1"
local new_version="$2"
local with_src="${3:-yes}" # yes/no — controls whether staging has src/
local extra_setup="${4:-}" # optional bash to run after setup
local sandbox=$(mktemp -d /tmp/dashcaddy-test-XXXXXX)
local dash_root="$sandbox/opt/dashcaddy"
mkdir -p "$dash_root"/{updates,bin,scripts}
mkdir -p "$dash_root/updates/staging/dashcaddy-api"
mkdir -p "$dash_root/dashcaddy-api/data"
# ── FROM deployment ──
echo '{"services":[]}' > "$dash_root/dashcaddy-api/data/services.json"
cat > "$dash_root/dashcaddy-api/server.js" << 'EOF'
const { createApp } = require('./src/app');
EOF
if [[ "$with_src" == "yes" ]]; then
mkdir -p "$dash_root/dashcaddy-api/src/managers"
cat > "$dash_root/dashcaddy-api/src/app.js" << 'EOF'
module.exports = { createApp: () => ({ app: {}, log: console, config: {} }) };
EOF
cat > "$dash_root/dashcaddy-api/src/managers/license-manager.js" << 'EOF'
const keygen = require('../../license-keygen');
module.exports = {};
EOF
fi
cat > "$dash_root/dashcaddy-api/license-keygen.js" << 'EOF'
module.exports = { verifyCode: () => true };
EOF
echo "from-commit" > "$dash_root/dashcaddy-api/VERSION"
# ── STAGING (new version) ──
cp "$dash_root/dashcaddy-api/server.js" "$dash_root/updates/staging/dashcaddy-api/"
cp "$dash_root/dashcaddy-api/license-keygen.js" "$dash_root/updates/staging/dashcaddy-api/"
if [[ "$with_src" == "yes" ]]; then
cp -r "$dash_root/dashcaddy-api/src" "$dash_root/updates/staging/dashcaddy-api/"
fi
echo "new-commit-$new_version" > "$dash_root/updates/staging/dashcaddy-api/VERSION"
# ── Mocked docker ──
cat > "$dash_root/bin/docker" << 'EOF'
#!/usr/bin/env bash
echo "[mock-docker] $*" >> "${MOCK_DOCKER_LOG:-/tmp/mock-docker.log}"
case "$1" in
build)
: >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-build-ran"
exit 0
;;
rm)
: >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-rm-ran"
exit 0
;;
run)
: >> "${IMAGE_MARKER_DIR:-/tmp}/.docker-run-ran"
exit 0
;;
compose|version)
exit 0
;;
*)
exit 0
;;
esac
EOF
chmod +x "$dash_root/bin/docker"
# Fake start.sh — NOT created in the sandbox so deploy_mode picks "run"
# (which exercises docker rm + docker run paths in restart_container).
# Production DNS2 has start.sh and uses the startsh deploy path; the test
# deliberately diverges so we observe the full docker restart sequence.
# ── Post-deploy verifier (real script copied in) ─────────────────────────
# dashcaddy-update.sh hard-codes /opt/dashcaddy/scripts/dashcaddy-post-deploy-patches.sh
# (which the sed rewrite maps to $dash_root/scripts/...). For the verifier to
# actually be invoked, we copy the real script into the sandbox. The verifier
# is the one being tested here; we want to observe its behavior end-to-end.
local verifier_src="${SCRIPT_DIR}/dashcaddy-post-deploy-patches.sh"
if [[ ! -f "$verifier_src" ]]; then
verifier_src="$(cd "${SCRIPT_DIR}/../../scripts" 2>/dev/null && pwd)/dashcaddy-post-deploy-patches.sh"
fi
if [[ -f "$verifier_src" ]]; then
cp "$verifier_src" "$dash_root/scripts/dashcaddy-post-deploy-patches.sh"
chmod +x "$dash_root/scripts/dashcaddy-post-deploy-patches.sh"
fi
# ── Path-rewritten update script ──
local patched="$sandbox/patched-update.sh"
sed "s|/opt/dashcaddy|$dash_root|g" "$UPDATE_SCRIPT_SRC" > "$patched"
chmod +x "$patched"
if [[ -n "$extra_setup" ]]; then
( cd "$sandbox" && eval "$extra_setup" )
fi
# Write a state file so the caller can recover the paths
cat > "$sandbox/.sandbox-paths" << EOF
SANDBOX_ROOT=$sandbox
DASH_ROOT=$dash_root
PATCHED_SCRIPT=$patched
API_SOURCE_DIR=$dash_root/dashcaddy-api
STAGING_DIR=$dash_root/updates/staging/dashcaddy-api
UPDATES_DIR=$dash_root/updates
EOF
echo "$sandbox/.sandbox-paths"
}
write_trigger() {
local updates_dir="$1" action="$2" to_version="$3" from_version="$4" staging_dir="$5" api_source_dir="$6"
cat > "$updates_dir/trigger.json" << EOF
{
"action": "${action}",
"version": "${to_version}",
"fromVersion": "${from_version}",
"channel": "stable",
"commit": "new-commit-${to_version}",
"stagingDir": "${staging_dir}",
"apiSourceDir": "${api_source_dir}"
}
EOF
}
load_paths() {
local paths_file="$1"
# shellcheck disable=SC1090
source "$paths_file"
}
cleanup_sandbox() {
local sandbox="$1"
rm -rf "$sandbox" /tmp/mock-docker.log 2>/dev/null
}
# ────────────────────────────────────────────────────────────────────────────
# SCENARIO 1: Happy path — update succeeds end-to-end
# ────────────────────────────────────────────────────────────────────────────
echo "=== Scenario 1: happy path — update v1.14.8 -> v1.14.9 ==="
PATHS=$(build_sandbox "1.14.8" "1.14.9" "yes")
SANDBOX=$(dirname "$PATHS")
load_paths "$PATHS"
write_trigger "$UPDATES_DIR" "update" "1.14.9" "1.14.8" "$STAGING_DIR" "$API_SOURCE_DIR"
RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \
IMAGE_MARKER_DIR="$SANDBOX" \
run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \
"$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc")
assert_eq "pipeline exit code" "0" "$RC"
assert_file_exists "result.json exists" "$UPDATES_DIR/result.json"
assert_json_field "result.success=true" "$UPDATES_DIR/result.json" "success" "True"
assert_json_field "result.version=1.14.9" "$UPDATES_DIR/result.json" "version" "1.14.9"
assert_file_exists "docker build ran" "$SANDBOX/.docker-build-ran"
assert_file_exists "docker rm ran" "$SANDBOX/.docker-rm-ran"
assert_file_exists "docker run ran" "$SANDBOX/.docker-run-ran"
assert_dir_exists "code backup dir created" "$UPDATES_DIR/backups/1.14.8"
assert_file_exists "code backup has server.js" "$UPDATES_DIR/backups/1.14.8/server.js"
assert_dir_exists "data backup dir created" "$UPDATES_DIR/backups/1.14.8/data-backup"
assert_dir_exists "update-state backup dir created" "$UPDATES_DIR/backups/1.14.8/update-state"
assert_file_exists "update-state backup has trigger.json.processing" "$UPDATES_DIR/backups/1.14.8/update-state/trigger.json.processing"
assert_eq "api source VERSION updated" "new-commit-1.14.9" "$(cat "$API_SOURCE_DIR/VERSION" 2>/dev/null)"
assert_grep "docker was invoked with build" "$SANDBOX/.docker-calls.log" "build -t dashcaddy-dashcaddy-api:latest"
assert_grep "docker was invoked with run" "$SANDBOX/.docker-calls.log" "run -d --restart unless-stopped"
assert_grep "pipeline log shows successful update" "$SANDBOX/pipeline.log" "Update successful"
cleanup_sandbox "$SANDBOX"
# ────────────────────────────────────────────────────────────────────────────
# SCENARIO 2: v1.14.4-style broken tarball (no src/) — verifier should fail
# the build. Pipeline exits non-zero, result.json reports failure.
#
# AS-OF-CURRENT dashcaddy-update.sh: the verifier's failure is logged as a
# WARNING and the build proceeds anyway (the script does not abort on verifier
# failure). Mocked docker build always succeeds, so the pipeline ends with
# success=true. The value of this scenario is asserting that the verifier IS
# invoked, DOES detect the v1.14.4-class bug, and emits the expected error
# message — i.e. the verifier itself works. Blocking the build on verifier
# failure is a separate gap in dashcaddy-update.sh (TODO: tighten the call
# site in main() so verifier failure aborts).
# ────────────────────────────────────────────────────────────────────────────
echo
echo "=== Scenario 2: v1.14.4-class bug (no src/ in staging) — verifier detects it ==="
PATHS=$(build_sandbox "1.14.4" "1.14.5" "no")
SANDBOX=$(dirname "$PATHS")
load_paths "$PATHS"
write_trigger "$UPDATES_DIR" "update" "1.14.5" "1.14.4" "$STAGING_DIR" "$API_SOURCE_DIR"
RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \
IMAGE_MARKER_DIR="$SANDBOX" \
run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \
"$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc")
# Current production behavior: verifier warns, build proceeds, pipeline succeeds.
assert_eq "pipeline exit code" "0" "$RC"
assert_file_exists "result.json exists" "$UPDATES_DIR/result.json"
assert_json_field "result.success=true (build proceeded despite verifier warning)" "$UPDATES_DIR/result.json" "success" "True"
# The KEY assertion: verifier actually caught the bug.
assert_grep "verifier detected the missing src/ tree" "$SANDBOX/pipeline.log" "Build should be ABORTED"
assert_grep "verifier failure was surfaced as a warning" "$SANDBOX/pipeline.log" "Post-deploy patches exited non-zero"
# Build still ran (current code ignores verifier failure).
assert_file_exists "docker build ran (current code proceeds past verifier failure)" "$SANDBOX/.docker-build-ran"
cleanup_sandbox "$SANDBOX"
# ────────────────────────────────────────────────────────────────────────────
# SCENARIO 3: Rollback — action=rollback restores from backup
# ────────────────────────────────────────────────────────────────────────────
echo
echo "=== Scenario 3: rollback — restore from backup directory ==="
PATHS=$(build_sandbox "1.14.9" "1.14.10" "yes")
SANDBOX=$(dirname "$PATHS")
load_paths "$PATHS"
# Pre-populate a backup dir (simulate that a prior update created it)
mkdir -p "$UPDATES_DIR/backups/1.14.8/data-backup"
echo '{"services":[]}' > "$UPDATES_DIR/backups/1.14.8/data-backup/services.json"
cat > "$UPDATES_DIR/backups/1.14.8/server.js" << 'EOF'
// ROLLBACK VERSION
const { createApp } = require('./src/app');
console.log('ROLLBACK-1.14.8');
EOF
echo "rollback-commit-1.14.8" > "$UPDATES_DIR/backups/1.14.8/VERSION"
mkdir -p "$UPDATES_DIR/backups/1.14.8/src"
cat > "$UPDATES_DIR/backups/1.14.8/src/app.js" << 'EOF'
module.exports = { createApp: () => ({ rollback: '1.14.8' }) };
EOF
cp "$UPDATES_DIR/backups/1.14.8/license-keygen.js" "$UPDATES_DIR/backups/1.14.8/" 2>/dev/null
# Rollback needs license-keygen.js in backup too
cat > "$UPDATES_DIR/backups/1.14.8/license-keygen.js" << 'EOF'
module.exports = { verifyCode: () => true };
EOF
# Write rollback trigger (no staging_dir needed for rollback)
cat > "$UPDATES_DIR/trigger.json" << EOF
{
"action": "rollback",
"version": "1.14.8",
"fromVersion": "1.14.9",
"channel": "stable",
"commit": "",
"stagingDir": "",
"apiSourceDir": "${API_SOURCE_DIR}"
}
EOF
RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \
IMAGE_MARKER_DIR="$SANDBOX" \
run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \
"$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc")
assert_eq "rollback pipeline exit code" "0" "$RC"
assert_file_exists "result.json exists" "$UPDATES_DIR/result.json"
assert_json_field "result.success=true" "$UPDATES_DIR/result.json" "success" "True"
assert_json_field "result.version=1.14.8" "$UPDATES_DIR/result.json" "version" "1.14.8"
assert_file_exists "docker build called (rollback rebuilds)" "$SANDBOX/.docker-build-ran"
assert_eq "api source VERSION restored" "rollback-commit-1.14.8" "$(cat "$API_SOURCE_DIR/VERSION" 2>/dev/null)"
assert_grep "pipeline log shows rollback" "$SANDBOX/pipeline.log" "ROLLBACK"
cleanup_sandbox "$SANDBOX"
# ────────────────────────────────────────────────────────────────────────────
# SCENARIO 4: No trigger file — pipeline exits cleanly without doing anything
# ────────────────────────────────────────────────────────────────────────────
echo
echo "=== Scenario 4: no trigger.json — exits 0 with no-op ==="
PATHS=$(build_sandbox "1.14.9" "1.14.10" "yes")
SANDBOX=$(dirname "$PATHS")
load_paths "$PATHS"
# Deliberately don't write trigger.json
RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \
IMAGE_MARKER_DIR="$SANDBOX" \
run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \
"$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc")
assert_eq "no-op exit code" "0" "$RC"
assert_grep "logs 'nothing to do'" "$SANDBOX/pipeline.log" "No trigger file found"
assert_not_exists "docker build did NOT run" "$SANDBOX/.docker-build-ran"
cleanup_sandbox "$SANDBOX"
# ────────────────────────────────────────────────────────────────────────────
# SCENARIO 5: Channel rejection — prerelease trigger on default host exits 1
# ────────────────────────────────────────────────────────────────────────────
echo
echo "=== Scenario 5: prerelease channel rejected (no ALLOW_PRERELEASE) ==="
PATHS=$(build_sandbox "1.14.9" "1.15.0-beta" "yes")
SANDBOX=$(dirname "$PATHS")
load_paths "$PATHS"
cat > "$UPDATES_DIR/trigger.json" << EOF
{
"action": "update",
"version": "1.15.0-beta",
"fromVersion": "1.14.9",
"channel": "beta",
"commit": "new-commit-1.15.0-beta",
"stagingDir": "${STAGING_DIR}",
"apiSourceDir": "${API_SOURCE_DIR}"
}
EOF
RC=$(MOCK_DOCKER_LOG="$SANDBOX/.docker-calls.log" \
IMAGE_MARKER_DIR="$SANDBOX" \
run_pipeline "$DASH_ROOT" "$PATCHED_SCRIPT" "yes" \
"$SANDBOX/pipeline.log" "$SANDBOX/pipeline.rc")
assert_eq "channel rejection exit code" "1" "$RC"
assert_file_exists "result.json exists" "$UPDATES_DIR/result.json"
assert_json_field "result.success=false" "$UPDATES_DIR/result.json" "success" "False"
assert_grep "result mentions channel rejection" "$UPDATES_DIR/result.json" "Channel 'beta' not allowed"
assert_not_exists "docker build did NOT run" "$SANDBOX/.docker-build-ran"
cleanup_sandbox "$SANDBOX"
# ────────────────────────────────────────────────────────────────────────────
# Summary
# ────────────────────────────────────────────────────────────────────────────
echo
echo "═══════════════════════════════════════════════════════════"
echo " dashcaddy-update.sh integration test"
echo " PASS: $pass FAIL: $fail"
echo "═══════════════════════════════════════════════════════════"
if (( fail > 0 )); then
exit 1
fi
echo "All scenarios passed."