dashcaddy/start.sh

#!/bin/bash
set -e
CONTAINER_NAME="dashcaddy-api"
IMAGE="dashcaddy-dashcaddy-api:latest"
DATA_DIR="/opt/dashcaddy/dashcaddy-api/data"
CADDYFILE="/etc/caddy/Caddyfile"
ASSETS_DIR="/var/www/dashcaddy-status/assets"
UPDATES_DIR="/opt/dashcaddy/updates"
BACKUPS_DIR="/opt/dashcaddy/backups"
HOST_IP="172.17.0.1"
# DC-118: this host's own routable IPs (comma-sep) — passed to the API so the
# caddy-event self-noise filter can drop the host's own curl probes (watchdog,
# cron) without blinding the store to external curl traffic. Loopback is
# always implicit in the worker; add the tailscale IP when discoverable.
SELF_IPS="127.0.0.1"
TS_IP="$(tailscale ip -4 2>/dev/null | head -1 || true)"
[ -n "$TS_IP" ] && SELF_IPS="${SELF_IPS},${TS_IP}"
# Local Technitium (binds 0.0.0.0:53) resolves *.sami + recurses for docker subnet
# external fallback. Without this the container only has 8.8.8.8 and every
# *.sami health-check probe fails with ENOTFOUND (uptime bars stay empty).
DNS_PRIMARY="100.121.150.22"   # Technitium (Tailscale IP) — resolves *.sami
# DC-121: the fallback must ALSO serve *.sami. Node's tls.connect resolves via
# dns.lookup → getaddrinfo → musl, which queries ALL resolv.conf nameservers in
# PARALLEL and takes the first reply. With 8.8.8.8 as fallback, Google NXDOMAINs
# the internal .sami TLD and wins that race ~2-5% of the time. Measured on DNS2
# inside the live container 2026-08-24: dns.lookup 18/400 ENOTFOUND for records
# that resolve fine via the primary; c-ares pinned to 8.8.8.8 alone returns
# NXDOMAIN 10/10; c-ares pinned to the primary 0/400. (Source of ssl-monitor
# "Failed to check cert" warn noise; the git.sami /etc/hosts pin below was a
# per-name paperover of this same class.) DNS1's Technitium secondary
# (100.71.97.12) serves *.sami AND recurses for external names — both verified
# from inside the container — so whichever resolver wins the race, the answer
# is correct.
DNS_FALLBACK="100.71.97.12"    # DNS1 Technitium secondary — serves *.sami + recurses

# --- DC-164 rev4: operator env overrides --------------------------------------
# /etc/dashcaddy/operator.env is forwarded into the container as -e flags by
# dashcaddy-api/scripts/operator-flags.js (Node; single O_NOFOLLOW fd pinned
# for BOTH the contract checks and the read — no TOCTOU window). Contract
# violation (rc 3) = fail closed: warn and forward NOTHING. Unexpected error
# (any other rc != 0) = fail loud: ABORT startup, do not silently run with
# an unverified operator file state. The canonical flag:
#   ALLOW_LIFETIME_LICENSE=true   on the CREATOR's host only. This is a
# client-side rail, not the security boundary — lifetime codes still must
# HMAC-verify against the creator-only signing secret (never shipped to
# customers), so flipping the flag on a customer host grants nothing.
# Missing file / no overrides = normal customer case (zero flags added).
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPERATOR_ENV_FILE="${OPERATOR_ENV_FILE:-/etc/dashcaddy/operator.env}"
OPERATOR_FLAGS=()
OP_HELPER="$SCRIPT_DIR/dashcaddy-api/scripts/operator-flags.js"
if [ -e "$OPERATOR_ENV_FILE" ]; then
  if [ ! -r "$OP_HELPER" ]; then
    echo "[start.sh] FATAL: operator.env present but contract checker missing: $OP_HELPER" >&2
    exit 1
  fi
  OP_NODE="${OPERATOR_NODE:-node}"
  if ! command -v "$OP_NODE" >/dev/null 2>&1; then
    echo "[start.sh] FATAL: operator.env present but '$OP_NODE' not found (set OPERATOR_NODE)" >&2
    exit 1
  fi
  OP_RAW="$( "$OP_NODE" "$OP_HELPER" "$OPERATOR_ENV_FILE" )" || OP_RC=$?
  OP_RC="${OP_RC:-0}"
  if [ "$OP_RC" -eq 3 ]; then
    echo "[start.sh] WARNING: operator env file rejected by contract check — forwarding NO overrides" >&2
  elif [ "$OP_RC" -ne 0 ]; then
    echo "[start.sh] FATAL: operator contract checker failed (exit $OP_RC) — aborting startup" >&2
    exit "$OP_RC"
  elif [ -n "$OP_RAW" ]; then
    while IFS= read -r kv; do
      [ -n "$kv" ] && OPERATOR_FLAGS+=(-e "$kv")
    done <<< "$OP_RAW"
    echo "[start.sh] operator.env: $(( ${#OPERATOR_FLAGS[@]} / 2 )) items forwarded (keys only: $(printf '%s\n' "${OPERATOR_FLAGS[@]}" | grep -v '^-e$' | cut -d= -f1 | paste -sd, -))"
  fi
fi

# --- One-time migration from Docker image layer to bind mount --------------
# DC-039 follow-up. Before v1.14.10, certain modules (audit-logger, license-
# keygen, credential-manager) defaulted their files to /app/src/* via
# path.join(__dirname, 'foo.json'). Those writes landed in the Docker image
# layer and VANISHED on every container recreate. This step scans for any
# non-empty zombie files left over from a previous image (where /opt/dashcaddy/
# previously used /opt/dashcaddy/dashcaddy-api/src/... as the path root) and
# copies their contents into the bind-mounted data dir ONCE.
#
# Idempotent: bails out if the migration sentinel file already exists.
# Designed to be a no-op on every fresh install.
MIGRATION_SENTINEL="${DATA_DIR}/.migrated-from-image-layer"
IMAGE_LAYER_ZOMBIES=(
  "/opt/dashcaddy/dashcaddy-api/src/security/audit-log.json"
  "/opt/dashcaddy/dashcaddy-api/src/security/.encryption-key"
  "/opt/dashcaddy/dashcaddy-api/src/security/.encryption-key.bak"
  "/opt/dashcaddy/dashcaddy-api/src/utils/error.log"
  "/opt/dashcaddy/dashcaddy-api/src/managers/.license-secret"
  "/opt/dashcaddy/dashcaddy-api/src/managers/.license-counter"
)
# Note: set -e is active at top of script. Each per-file step uses an
# explicit `|| true` (or guarded `if`) so a single unreadable zombie file
# can't take down the whole container. The sentinel write at the end is
# outside any conditional so it always runs once.
run_image_layer_migration() {
  if [ -f "${MIGRATION_SENTINEL}" ]; then
    return 0
  fi
  mkdir -p "${DATA_DIR}" || { echo "[start.sh] [migration] mkdir failed: ${DATA_DIR}"; return 0; }
  local migrated=0
  for src in "${IMAGE_LAYER_ZOMBIES[@]}"; do
    if [ -f "${src}" ] && [ -s "${src}" ]; then
      local dest_name dest
      dest_name="$(basename "${src}")"
      dest="${DATA_DIR}/migrated-${dest_name}"
      if [ ! -f "${dest}" ]; then
        echo "[start.sh] [migration] Recovering image-layer file: ${src} -> ${dest}"
        if cp -a "${src}" "${dest}" 2>/dev/null; then
          migrated=$((migrated + 1))
        else
          echo "[start.sh] [migration] WARN: failed to copy ${src} (continuing)"
        fi
      fi
    fi
  done
  if [ "$migrated" -gt 0 ]; then
    echo "[start.sh] [migration] Recovered ${migrated} file(s) from image layer."
    echo "[start.sh] [migration] Review files prefixed 'migrated-' in ${DATA_DIR} and merge or delete."
  fi
  # Sentinel write MUST run regardless of any per-file failure above.
  date -u +%Y-%m-%dT%H:%M:%SZ > "${MIGRATION_SENTINEL}" 2>/dev/null || echo "1" > "${MIGRATION_SENTINEL}"
}
run_image_layer_migration

# --- /etc/hosts overrides for the container ---------------------------------
# The base image (node:20-alpine) has no entries for *.sami. We must inject
# them via --add-host so health checks inside the container can resolve LAN
# and Tailscale IPs to the right destinations.
#
# IMPORTANT: Do NOT add `git.sami:100.81.59.99` (DNS3). DNS3 does not serve
# HTTPS on 443 — it only serves Gitea on :3030. Setting git.sami → DNS3 in
# the container would make health checks bypass Caddy and hit a closed port.
# Let Caddy (on DNS2:443) handle git.sami and route to DNS3:3030 internally.
#
# Layout:
#   dns3.sami / gitea → DNS3 (Tailscale IP, used by tools inside the container
#                        that need to talk to Gitea directly, e.g. backups)
#   dns3-wan.sami     → DNS3 Contabo WAN fallback
#   dns2.sami         → DNS2 (this host) — for cross-service references
#   dns1.sami         → DNS1 (SAMI-CLOUD-U32)
#   dc-contabo-de     → DashCaddy Contabo test instance
#   git.dashcaddy.net → DashCaddy upstream git
#   git.sami          → DNS2 (NOT DNS3 — see warning above). Resolves an
#                       intermittent ENOTFOUND in the ssl-monitor's TLS
#                       handshake check (~2/h) by pinning the name in the
#                       container's /etc/hosts to the Caddy listener.
#   ca.sami           → local CA (DN2 + DN3 both have their own)
ADD_HOST_FLAGS=(
  # host.docker.internal → host bridge IP (Docker host-gateway). The caddy
  # upstream watcher probes Caddy site upstreams from INSIDE this container;
  # `reverse_proxy localhost:PORT` in a site file means the HOST's loopback,
  # so the watcher remaps loopback probe targets to this name (see
  # dashcaddy-api/src/monitoring/caddy-upstream-watcher.js). Without this
  # entry the probes would hit the container's own loopback and report every
  # host-side upstream as dead.
  --add-host=host.docker.internal:host-gateway
  --add-host=dns3.sami:100.81.59.99
  --add-host=gitea:100.81.59.99
  --add-host=dns3-wan.sami:74.208.167.19
  --add-host=dns2.sami:100.121.150.22
  --add-host=dns1.sami:100.71.97.12
  --add-host=dc-contabo-de:100.98.123.59
  --add-host=git.dashcaddy.net:100.98.123.59
  --add-host=git.sami:100.121.150.22
  # ca.sami resolves via DNS to 100.121.150.22 (Caddy on DNS2). Don't pin
  # to 127.0.0.1 — nothing listens on 443 inside the container, so the
  # health checker would fail with ECONNREFUSED. The CA itself is a
  # public-facing service that goes through Caddy just like every other *.sami.
)

# Always recreate to ensure env vars are correct (CONFIG_FILE defaults to /etc/dashcaddy/ which doesn't exist)
if docker ps -a --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then
  echo "[start.sh] Recreating container to apply correct env vars..."
  docker rm -f ${CONTAINER_NAME}
fi

# Tailscale CLI + control socket — lets the container invoke
# `tailscale status --json` to populate /api/v1/tailscale/status etc.
# The binary is statically linked (Go), so the bind-mount works under
# the container's Alpine libc without any library forwarding.
# Both mounts are read-only: `tailscale status --json` is a read query
# that the local tailscaled handles; we never need to mutate state
# from inside the container.
echo "[start.sh] Creating container with full config..."

# Sync the freshly-built dashboard bundle into the static directory Caddy
# serves — decided by VERSION METADATA, not file mtimes (mtimes are not
# reliable: scp/tar/cp can preserve or shuffle them). DC-122 contract:
#   - The self-updater writes a STAMP (update-stamp.json) into the live web
#     root when it deploys a frontend; while that stamp is newer than the
#     source tree's VERSION file, start.sh must NOT touch the live bundle.
#   - Normal builds: publishing bumps the source VERSION (mtime = build time)
#     and clears any stale stamp, so source wins and the sync happens.
echo "[start.sh] Syncing dashboard bundle into static dir (metadata-driven)..."
mkdir -p /var/www/dashcaddy-status/dist
if [ -d /opt/dashcaddy/status/dist ]; then
  NEEDS_SYNC=1
  STAMP=/var/www/dashcaddy-status/update-stamp.json
  SRC_VERSION=/opt/dashcaddy/dashcaddy-api/VERSION
  if [ -f "$STAMP" ] && [ -f "$SRC_VERSION" ] && [ "$STAMP" -nt "$SRC_VERSION" ]; then
    # A self-updater deployment is newer than the last source build: hands off.
    echo "[start.sh] Deployed frontend stamp newer than source VERSION — skipping sync to preserve deployed frontend."
    NEEDS_SYNC=0
  fi
  if [ "$NEEDS_SYNC" = "1" ]; then
    cp /opt/dashcaddy/status/dist/*.js /var/www/dashcaddy-status/dist/ 2>/dev/null || true
    cp /opt/dashcaddy/status/sw.js /var/www/dashcaddy-status/ 2>/dev/null || true
    cp /opt/dashcaddy/status/index.html /var/www/dashcaddy-status/ 2>/dev/null || true
    rm -f "$STAMP"
    echo "[start.sh] Bundle synced ($(ls /opt/dashcaddy/status/dist/*.js 2>/dev/null | wc -l) bundle files + sw.js + index.html)."
  fi
else
  echo "[start.sh] WARN: /opt/dashcaddy/status/dist missing — skipping sync (frontend will be stale)."
fi

docker run -d --restart unless-stopped --name ${CONTAINER_NAME} \
  --memory=1g --memory-swap=2g --cpus=2 \
  --log-driver json-file --log-opt max-size=10m --log-opt max-file=3 \
  --add-host=get.dashcaddy.net:194.233.88.206 \
  --add-host=get2.dashcaddy.net:194.233.88.206 \
  --dns ${DNS_PRIMARY} \
  --dns ${DNS_FALLBACK} \
  "${ADD_HOST_FLAGS[@]}" \
  -p 127.0.0.1:3001:3001 \
  -v ${DATA_DIR}:/app/data \
  -v ${BACKUPS_DIR}:/app/backups \
  -v ${CADDYFILE}:/caddyfile \
  -v /etc/caddy/sites:/etc/caddy/sites:ro \
  -v /var/log/caddy:/var/log/caddy:ro \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v ${ASSETS_DIR}:/app/assets \
  -v ${UPDATES_DIR}:/app/updates \
  -v /opt/sami-files/logs:/opt/sami-files/logs:ro \
  -v /usr/bin/tailscale:/usr/bin/tailscale:ro \
  -v /var/run/tailscale:/var/run/tailscale:ro \
  -v /etc/ssl/sami-ca:/etc/ssl/sami-ca:ro \
  -v /var/log/journal:/var/log/journal:ro \
  -v /usr/bin/journalctl:/usr/bin/journalctl:ro \
  -e NODE_ENV=production \
  -e SERVICES_FILE=/app/data/services.json \
  -e CONFIG_FILE=/app/data/config.json \
  -e BACKUP_DIR=/app/backups \
  -e DNS_CREDENTIALS_FILE=/app/data/dns-credentials.json \
  -e CREDENTIALS_FILE=/app/data/credentials.json \
  -e ENCRYPTION_KEY_FILE=/app/data/.encryption-key \
  -e HEALTH_HISTORY_FILE=/app/data/health-history.json \
  -e HEALTH_CONFIG_FILE=/app/data/health-config.json \
  -e CADDYFILE_PATH=/caddyfile \
  -e CADDY_ADMIN_URL=http://${HOST_IP}:2019 \
  -e CADDY_ACCESS_LOG=/var/log/caddy/access.log \
  -e DASHCADDY_SELF_IPS="${SELF_IPS}" \
  -e ASSETS_DIR=/app/assets \
  -e DASHCADDY_API_SOURCE_DIR=/opt/dashcaddy/dashcaddy-api \
  -e DASHCADDY_UPDATE_ENABLED=true \
  -e SHIPDECK_BRIDGE_URL=http://172.17.0.1:8977 \
  -e SHIPDECK_BRIDGE_TOKEN_FILE=/app/data/shipdeck-bridge-token \
  -e CA_CERT_PATH=/etc/ssl/sami-ca/root.crt \
  ${OPERATOR_FLAGS[@]+"${OPERATOR_FLAGS[@]}"} \
  ${IMAGE}