diff --git a/.gitignore b/.gitignore index 60db35d..cf05d41 100644 --- a/.gitignore +++ b/.gitignore @@ -8,70 +8,22 @@ dashcaddy-api/credentials.json dashcaddy-api/.env .env dashcaddy-api/alert-config.json -dashcaddy-api/audit-log.json -dashcaddy-api/audit-log.json.lock -dashcaddy-api/backup-config.json -dashcaddy-api/backup-history.json -dashcaddy-api/container-stats.json -dashcaddy-api/health-config.json -dashcaddy-api/health-history.json -dashcaddy-api/update-config.json -dashcaddy-api/update-history.json -dashcaddy-api/dashcaddy-errors.log -# Auto-updater backups (created by dashcaddy-update.sh when rolling back) -start.sh.bak* -scripts/*.bak* +# Build artifacts +*.log +*.tar.gz -# Auto-updater runtime state (history + secrets + staging) +# Local artifacts +CLAUDE.md + +# Runtime state directories +backups/ updates/ -# Scratch / debug scripts (left over from past sessions) -cm_check*.js -full_test.js -login_test.js -login_backup_test.js - -# Build output -dashcaddy-installer/build-output/ -dashcaddy-installer/dist/ -status/dist/ - -# Vendor / third-party -status/vendor/ - -# Backup files -*.backup.html -*.backup.*.html -*.recovered -backups/ - -# IDE / editor -.claude/ -.kiro/ -.vscode/ - -# Session-specific docs (not project docs) -DEPLOYMENT-SUCCESS.md -FINAL-DEPLOYMENT-REPORT.md -TEST-RESULTS.md -TESTING-GUIDE.md -DashCA-Plan.md -vhdx-cleanup-instructions.md -DESLOPIFICATION-ROADMAP.md -SECURITY-IMPROVEMENTS.md -WHAT-IS-DASHCADDY.md -error-handling-cleanup-summary.md -error-handling-migration-complete.md - -# Utility scripts (local only) -check-e.ps1 -disk-scan.ps1 -disk-scan2.ps1 -fix-wsl-and-mount.ps1 -fix-ctx-routes.sh -import-services.js - -# OS files -Thumbs.db -.DS_Store +# Generated post-deploy patch artifacts — flat copies of src/ files placed +# in dashcaddy-api/ root by scripts/dashcaddy-post-deploy-patches.sh to work +# around broken upstream tarballs. Real source lives in dashcaddy-api/src/. +# Once v1.15.0 ships src/ properly, these become obsolete. +dashcaddy-api/*.js +!dashcaddy-api/license-keygen.js +!dashcaddy-api/platform-paths.js \ No newline at end of file diff --git a/BUILD-PIPELINE-FIX.md b/BUILD-PIPELINE-FIX.md new file mode 100644 index 0000000..d7eb418 --- /dev/null +++ b/BUILD-PIPELINE-FIX.md @@ -0,0 +1,172 @@ +# Build Pipeline Fix — Complete Source in Tarballs + +**Date:** 2026-07-01 +**Bug:** Every published release tarball at `get.dashcaddy.net/release/` was missing `dashcaddy-api/src/` — the directory holding ~80% of the application code (app.js, all managers, monitoring, docker, security, utilities modules). Hosts had to run a post-deploy patches script after every update to fix 23+ broken `require('./src/...')` paths. + +--- + +## What was broken + +`/opt/dashcaddy-release/build-release.sh` (the script triggered by the Gitea webhook on push to `main`) assembled the tarball using these copy commands: + +```bash +cp -f dashcaddy-api/*.js "$staging/dashcaddy-api/" # root-level only +cp -rf dashcaddy-api/routes/* "$staging/dashcaddy-api/routes/" +cp -f dashcaddy-api/package.json ... # misc root files +``` + +It never copied `dashcaddy-api/src/`, even though `server.js` does: + +```js +const { createApp } = require('./src/app'); +const authManager = require('./src/managers/auth-manager'); +const selfUpdater = require('./src/docker/self-updater'); +const healthChecker = require('./src/monitoring/health-checker'); +// ...and 20+ more require('./src/...') calls +``` + +**Result:** every published tarball was missing 60+ source files. The post-deploy script `dashcaddy-post-deploy-patches.sh` existed only to paper over this gap. + +The shipped tarball filename pattern (`dashcaddy-${version}.tar.gz`), the webroot path (`/var/www/get.dashcaddy.net/release/`), and existing `version.json` field names were preserved — only an additive fix. + +--- + +## What changed + +### 1. `build-release.sh` — tarball assembly (lines 45–63) + +Added three copy blocks after the existing API files section: + +```bash +# Application source (this is the bulk of the code: app.js, managers, monitoring, etc.) +if [ -d "dashcaddy-api/src" ]; then + cp -rf dashcaddy-api/src "$staging/dashcaddy-api/" +else + log "FATAL: dashcaddy-api/src/ not found in repo — refusing to build incomplete tarball" + exit 1 +fi + +# Optional app assets / scripts if they exist +[ -d "dashcaddy-api/assets" ] && cp -rf dashcaddy-api/assets "$staging/dashcaddy-api/" +[ -d "dashcaddy-api/scripts" ] && cp -rf dashcaddy-api/scripts "$staging/dashcaddy-api/" +``` + +Also simplified the routes copy from `cp -rf dashcaddy-api/routes/*` to `cp -rf dashcaddy-api/routes` — the previous form silently dropped dotfiles/hidden routes and would fail entirely on an empty directory under `set -e`. + +### 2. `build-release.sh` — verification step (lines 83–88) + +After the tarball is built, a self-check refuses to publish if `src/` isn't in it: + +```bash +if ! tar tzf "$tarball" | grep -q "^dashcaddy/dashcaddy-api/src/"; then + log "FATAL: tarball is missing dashcaddy-api/src/ — refusing to publish" + exit 1 +fi +log "Tarball contains src/: OK" +``` + +This makes the missing-src bug structurally impossible to recur. + +### 3. `build-release.sh` — `src_sha256` field (lines 95–99, 108) + +Added computation of a deterministic SHA-256 over the `src/` directory contents (files in sorted order, hashed with sha256sum, then the resulting block rehashed): + +```bash +src_sha256=$(cd "$BUILD_DIR/repo" && find dashcaddy-api/src -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) +``` + +This is written into `version.json` as a new `src_sha256` field alongside the existing `sha256` (tarball hash). The self-updater at `dashcaddy-api/src/docker/self-updater.js` can now compare its locally-extracted `src/` hash to the remote `src_sha256` and detect drift between tarball-level metadata and actual source contents. + +`version.json` schema after the change: + +```json +{ + "version": "1.14.6", + "commit": "abc1234", + "date": "2026-07-01T08:45:52Z", + "sha256": "", + "src_sha256": "", + "changelog": "...", + "breaking": false, + "tarball": "dashcaddy-1.14.6.tar.gz" +} +``` + +`src_sha256` is **additive only** — no existing field was renamed or removed. + +### 4. Idempotency & safety + +- `set -euo pipefail` preserved. +- All new copies are guarded (`[ -d ... ]` for optional dirs; explicit `if [ -d ... ]` for `src/` with a fatal exit). +- Tarball filename pattern (`dashcaddy-${version}.tar.gz`) unchanged. +- Webroot path (`/var/www/get.dashcaddy.net/release/`) unchanged. +- Mirror rsync step unchanged — destination server will receive the new (complete) tarballs automatically. + +--- + +## How to verify locally + +The script can be smoke-tested without contacting Gitea or the mirror: + +```bash +# 1. Snapshot the repo into a scratch dir (avoid touching /opt/dashcaddy) +mkdir -p /tmp/verify/repo +tar --exclude='.git' --exclude='updates' --exclude='backups' \ + -C /opt/dashcaddy -cf - . | tar -C /tmp/verify/repo -xf - + +# 2. Replicate the assembly from build-release.sh against the snapshot +cd /tmp/verify/repo +mkdir -p /tmp/verify/dashcaddy/dashcaddy-api/routes /tmp/verify/dashcaddy/status /tmp/verify/dashcaddy/scripts +STG=/tmp/verify/dashcaddy +cp -f dashcaddy-api/*.js "$STG/dashcaddy-api/" 2>/dev/null || true +cp -rf dashcaddy-api/routes "$STG/dashcaddy-api/" +cp -f dashcaddy-api/package.json "$STG/dashcaddy-api/" +cp -f dashcaddy-api/package-lock.json "$STG/dashcaddy-api/" 2>/dev/null || true +cp -f dashcaddy-api/Dockerfile "$STG/dashcaddy-api/" +cp -f dashcaddy-api/openapi.yaml "$STG/dashcaddy-api/" 2>/dev/null || true +[ -d dashcaddy-api/src ] && cp -rf dashcaddy-api/src "$STG/dashcaddy-api/" +[ -d dashcaddy-api/assets ] && cp -rf dashcaddy-api/assets "$STG/dashcaddy-api/" +[ -d dashcaddy-api/scripts ] && cp -rf dashcaddy-api/scripts "$STG/dashcaddy-api/" +# ... status/ + scripts/ as in build-release.sh ... + +# 3. Build the tarball and run the verification step +cd /tmp/verify +tar czf test.tar.gz dashcaddy/ +tar tzf test.tar.gz | grep -q "^dashcaddy/dashcaddy-api/src/" && echo "src/ present: OK" + +# 4. Confirm src_sha256 is deterministic +find dashcaddy-api/src -type f | sort | xargs sha256sum | sha256sum +``` + +Expected output: + +- `src/ present: OK` +- `src_sha256` identical across two runs (no timestamps or non-deterministic ordering). + +The local dry-run on 2026-07-01 produced an 18 MB tarball with **74 `src/` entries** (was 0 before), and verified that all of `src/app.js`, `src/docker/self-updater.js`, `src/managers/auth-manager.js`, `src/managers/resource-monitor.js`, `src/monitoring/health-checker.js`, `src/utilities/startup-validator.js`, and `src/utils/http.js` are present. + +--- + +## Migration note for existing installations + +Hosts already running the old (src-less) release format will need to pick up one of the new tarballs to get the complete source tree: + +- **Option A (recommended):** trigger a normal update from `get.dashcaddy.net/release/latest.tar.gz`. Because the new tarball includes `src/`, no post-deploy patching is needed — `server.js` will resolve every `require('./src/...')` directly. The post-deploy-patches.sh script remains in place and is still safe to run (it's a no-op on a complete tree). +- **Option B (no network):** leave the host on its current release. The post-deploy-patches.sh script continues to function as before — it patches the broken `require()` paths after every update. Nothing changes for offline hosts. + +There is no database migration, no config-file change, and no restart ordering change required. The next tarball published after this commit will simply contain the missing `src/` directory. + +--- + +## Files modified + +| Path | Change | +|---|---| +| `/opt/dashcaddy-release/build-release.sh` | Added `src/`, `assets/`, `scripts/` copies + verification step + `src_sha256` field | +| `/opt/dashcaddy/BUILD-PIPELINE-FIX.md` | This document | + +## Files NOT modified (and why) + +- `dashcaddy-api/src/docker/self-updater.js` — `src_sha256` is now published in `version.json`, but the self-updater doesn't need a code change to *receive* it. Adding the comparison logic in the updater is a separate, optional task that should be done when ready to consume the new field. +- `dashcaddy-post-deploy-patches.sh` — kept as a safety net; now a no-op for fresh installs but still useful for legacy hosts. +- Any `version.json` already on disk at `/var/www/get.dashcaddy.net/release/` — overwritten automatically on the next release build. \ No newline at end of file diff --git a/scripts/dashcaddy-post-deploy-patches.sh b/scripts/dashcaddy-post-deploy-patches.sh new file mode 100755 index 0000000..342f4e1 --- /dev/null +++ b/scripts/dashcaddy-post-deploy-patches.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# DashCaddy Post-Deploy Patch Script +# Runs AFTER the host-side update script copies staging files into the API source +# directory, but BEFORE the Docker build. Fixes upstream bugs in the released +# tarball so the build succeeds and the container starts cleanly. +# +# Why this exists: +# v1.14.4 (commit d2a48b1) shipped with broken relative require paths: +# - Root server.js: `require('../src/...')` instead of `require('./src/...')` +# - Many src/**/*.js: `require('./module-name')` instead of +# `require('../module-name')` (files were moved into src/ but requires +# not updated to point at root-level modules) +# - Missing license-keygen.js at root +# Without these patches, every auto-update results in a crash-looping container. +# +# Idempotent: safe to run multiple times, only changes files that match the +# broken pattern. Reports what was already OK so you can confirm health. +# +# Usage: dashcaddy-post-deploy-patches.sh +# api_source_dir: e.g. /opt/dashcaddy/dashcaddy-api + +set -uo pipefail + +API_DIR="${1:-/opt/dashcaddy/dashcaddy-api}" + +log() { echo "[dashcaddy-patch] $(date '+%Y-%m-%d %H:%M:%S') $*"; } + +if [[ ! -d "$API_DIR" ]]; then + log "ERROR: API source directory not found: $API_DIR" + exit 1 +fi + +cd "$API_DIR" || exit 1 + +TOTAL_PATCHED=0 +TOTAL_ALREADY_OK=0 + +# ──────────────────────────────────────────────────────────────────────────── +# Patch 1: Root-level server.js — fix '../src/...' requires to './src/...' +# v1.14.4 was tagged with broken relative paths. server.js sits at the API +# root, so any `require('../src/...')` is one directory too high. +# ──────────────────────────────────────────────────────────────────────────── +SERVER_JS="$API_DIR/server.js" +if [[ -f "$SERVER_JS" ]]; then + if grep -q "require('\.\./src/" "$SERVER_JS"; then + BAD_COUNT=$(grep -c "require('\.\./src/" "$SERVER_JS" || true) + sed -i "s|require('\.\./src/|require('./src/|g" "$SERVER_JS" + if ! grep -q "require('\.\./src/" "$SERVER_JS"; then + log "Patched server.js: rewrote ${BAD_COUNT} '../src/...' requires to './src/...'" + TOTAL_PATCHED=$((TOTAL_PATCHED + BAD_COUNT)) + else + log "WARNING: server.js sed did not remove all bad requires" + fi + else + TOTAL_ALREADY_OK=$((TOTAL_ALREADY_OK + 1)) + log "server.js: already correct (no '../src/...' requires)" + fi +else + log "WARNING: $SERVER_JS not found" +fi + +# ──────────────────────────────────────────────────────────────────────────── +# Patch 2: src/managers/license-manager.js — fix './license-keygen' require +# The license-keygen module lives at the API root, so from src/managers/ +# the correct relative path is '../../license-keygen'. +# ──────────────────────────────────────────────────────────────────────────── +LICENSE_MGR="$API_DIR/src/managers/license-manager.js" +if [[ -f "$LICENSE_MGR" ]]; then + if grep -q "require('\./license-keygen')" "$LICENSE_MGR"; then + sed -i "s|require('\./license-keygen')|require('../../license-keygen')|g" "$LICENSE_MGR" + if grep -q "require('\.\./\.\./license-keygen')" "$LICENSE_MGR"; then + log "Patched license-manager.js: './license-keygen' → '../../license-keygen'" + TOTAL_PATCHED=$((TOTAL_PATCHED + 1)) + else + log "WARNING: license-manager.js sed did not apply" + fi + else + TOTAL_ALREADY_OK=$((TOTAL_ALREADY_OK + 1)) + log "license-manager.js: already correct" + fi +else + log "WARNING: $LICENSE_MGR not found" +fi + +# ──────────────────────────────────────────────────────────────────────────── +# Patch 3: Generic src/**/*.js require path fix +# For every file in any src/ subdir, find `require('./module-name')` patterns +# where module-name.js exists at API root but NOT in the same subdir, and +# rewrite them to `require('../module-name')`. +# +# This catches the bulk of v1.14.4's broken paths that the upstream refactor +# left behind (files moved into src/ but requires not updated). +# ──────────────────────────────────────────────────────────────────────────── +log "Scanning src/ for broken root-level requires..." + +GENERIC_PATCHED=0 +GENERIC_ALREADY_OK=0 + +# Build a list of all .js files in src/ (excluding tests) +while IFS= read -r -d '' src_file; do + # Get the directory containing this file relative to API_DIR + rel_dir=$(dirname "${src_file#$API_DIR/}") # e.g. "src/docker" + depth=$(echo "$rel_dir" | tr '/' '\n' | wc -l) + # depth=1 means src/foo.js (parent is "src") + # depth=2 means src/docker/foo.js (parent is "src/docker"), need ../ + + # Find all `require('./name')` patterns in this file + while IFS= read -r require_line; do + # Extract the module path from inside the quotes + mod_path=$(echo "$require_line" | grep -oE "require\(['\"]\./[a-zA-Z0-9_-]+['\"]\)" | head -1 | sed -E "s|require\(['\"]\./||; s|['\"]\)||") + + if [[ -z "$mod_path" ]]; then continue; fi + + # Compute the absolute path Node would resolve `./mod_path` to from this file + # Candidate 1: same dir, .js file + candidate="$rel_dir/$mod_path.js" + if [[ -f "$candidate" ]]; then + # File exists in same subdir → require is correct as-is + continue + fi + + # Candidate 2: same dir, directory with index.js + if [[ -d "$rel_dir/$mod_path" && -f "$rel_dir/$mod_path/index.js" ]]; then + continue + fi + + # Check if it exists at the root (one level above src/, or at the + # appropriate depth for nested src/ subdirs) + # For a file at $rel_dir/$file.js, './mod' resolves to $rel_dir/mod.js + # We need to find where mod.js actually exists. + + found_path="" + # Walk up from the same-dir candidate, checking each parent dir. + # Start by checking the file's own dir (already done above), then + # dirname(rel_dir), dirname(dirname(rel_dir)), ..., until we hit ".". + # rel_dir is relative to API_DIR, so when test_dir becomes ".", we + # should check API_DIR/$mod_path.js (the root), THEN break. + test_dir="$rel_dir" + while true; do + test_dir=$(dirname "$test_dir") + # Check this directory for the module: either .js file or dir/index.js + if [[ -f "$test_dir/$mod_path.js" || ( -d "$test_dir/$mod_path" && -f "$test_dir/$mod_path/index.js" ) ]]; then + found_path="$test_dir/$mod_path" + break + fi + # Stop when we've gone past root + [[ "$test_dir" == "." || "$test_dir" == "/" ]] && break + done + + if [[ -z "$found_path" ]]; then + # Module not found anywhere — leave it alone, would need investigation + continue + fi + + # Found at root. Compute the correct relative path from this file to root. + # For src/docker/self-updater.js requiring platform-paths (at root): + # need: '../../platform-paths' + file_dir=$(dirname "$src_file") + file_dir_rel="${file_dir#$API_DIR/}" # e.g. "src/docker" + + # Number of dirs to go up: count slashes + 1 + # "src/docker" → 2 dirs → go up 2: ../../platform-paths + up_count=$(echo "$file_dir_rel" | awk -F'/' '{print NF}') + + up_path="" + for ((i=0; i/dev/null || true) +done < <(find "$API_DIR/src" -type f -name "*.js" -not -path "*/node_modules/*" -not -path "*/__tests__/*" -print0 2>/dev/null) + +if [[ $GENERIC_PATCHED -gt 0 ]]; then + log "Generic src/ require patches: ${GENERIC_PATCHED} fixed" +else + log "Generic src/ require patches: 0 needed (all correct)" +fi +TOTAL_PATCHED=$((TOTAL_PATCHED + GENERIC_PATCHED)) +TOTAL_ALREADY_OK=$((TOTAL_ALREADY_OK + GENERIC_ALREADY_OK)) + +# ──────────────────────────────────────────────────────────────────────────── +# Patch 4: Ensure license-keygen.js exists at API root +# v1.14.4's tarball didn't ship the root-level license-keygen.js. If missing, +# restore from src/managers/license-keygen.js or a backup. +# ──────────────────────────────────────────────────────────────────────────── +LICENSE_ROOT="$API_DIR/license-keygen.js" +LICENSE_SRC="$API_DIR/src/managers/license-keygen.js" +if [[ ! -f "$LICENSE_ROOT" ]]; then + BACKUP_FILE="" + # Prefer the v1.13.x backup if it exists (the version that had it at root) + if [[ -d "$API_DIR/../updates/backups" ]]; then + BACKUP_FILE=$(find "$API_DIR/../updates/backups" -name "license-keygen.js" 2>/dev/null | head -1) + fi + # Fall back to src/managers/ if newer refactor put it there + if [[ -z "$BACKUP_FILE" && -f "$LICENSE_SRC" ]]; then + BACKUP_FILE="$LICENSE_SRC" + fi + # Last resort: search elsewhere + if [[ -z "$BACKUP_FILE" ]]; then + BACKUP_FILE=$(find /opt/dashcaddy -name "license-keygen.js" -not -path "*/node_modules/*" -not -path "*/updates/*" -not -path "*/backups/staging-*" 2>/dev/null | head -1) + fi + if [[ -n "$BACKUP_FILE" && -f "$BACKUP_FILE" ]]; then + cp -f "$BACKUP_FILE" "$LICENSE_ROOT" + log "Restored missing license-keygen.js from $BACKUP_FILE" + TOTAL_PATCHED=$((TOTAL_PATCHED + 1)) + else + log "ERROR: license-keygen.js missing at root and no backup available — build may fail" + fi +else + TOTAL_ALREADY_OK=$((TOTAL_ALREADY_OK + 1)) +fi + +# ──────────────────────────────────────────────────────────────────────────── +# Summary +# ──────────────────────────────────────────────────────────────────────────── +log "=== Post-deploy patch summary: ${TOTAL_PATCHED} require fixes applied, ${TOTAL_ALREADY_OK} components already OK ===" +exit 0 \ No newline at end of file