From 954be9e86846ef497b39fb1d58436b00b719cb06 Mon Sep 17 00:00:00 2001 From: Hermes Date: Wed, 10 Jun 2026 14:43:46 -0700 Subject: [PATCH] feat: auto-restart policies, SSL monitoring, DNS propagation, dependency tracking, config drift detection --- dashcaddy-api/VERSION | 2 +- dashcaddy-api/auto-restart-manager.js | 503 ++++++++++++++++++++ dashcaddy-api/config-drift-detector.js | 376 +++++++++++++++ dashcaddy-api/dependency-manager.js | 605 +++++++++++++++++++++++++ dashcaddy-api/dns-propagation.js | 273 +++++++++++ dashcaddy-api/package.json | 2 +- dashcaddy-api/routes/auto-restart.js | 164 +++++++ dashcaddy-api/routes/config-drift.js | 92 ++++ dashcaddy-api/routes/dependencies.js | 235 ++++++++++ dashcaddy-api/routes/dns.js | 74 ++- dashcaddy-api/routes/events.js | 45 +- dashcaddy-api/routes/ssl-monitor.js | 113 +++++ dashcaddy-api/src/app.js | 76 +++- dashcaddy-api/ssl-monitor.js | 411 +++++++++++++++++ scripts/backup-gitea-to-dropbox.sh | 83 ++++ 15 files changed, 3048 insertions(+), 6 deletions(-) create mode 100644 dashcaddy-api/auto-restart-manager.js create mode 100644 dashcaddy-api/config-drift-detector.js create mode 100644 dashcaddy-api/dependency-manager.js create mode 100644 dashcaddy-api/dns-propagation.js create mode 100644 dashcaddy-api/routes/auto-restart.js create mode 100644 dashcaddy-api/routes/config-drift.js create mode 100644 dashcaddy-api/routes/dependencies.js create mode 100644 dashcaddy-api/routes/ssl-monitor.js create mode 100644 dashcaddy-api/ssl-monitor.js create mode 100644 scripts/backup-gitea-to-dropbox.sh diff --git a/dashcaddy-api/VERSION b/dashcaddy-api/VERSION index 27f9cd3..f8e233b 100644 --- a/dashcaddy-api/VERSION +++ b/dashcaddy-api/VERSION @@ -1 +1 @@ -1.8.0 +1.9.0 diff --git a/dashcaddy-api/auto-restart-manager.js b/dashcaddy-api/auto-restart-manager.js new file mode 100644 index 0000000..3ca79cb --- /dev/null +++ b/dashcaddy-api/auto-restart-manager.js @@ -0,0 +1,503 @@ +/** + * Auto-Restart Manager - Per-container restart policies with retry tracking + * + * When a container goes down, attempts automatic restart up to N times + * (configurable per-service). Sends notifications on each attempt and + * when max retries are exceeded. Integrates with HealthChecker events. + * + * @module auto-restart-manager + */ + +const EventEmitter = require('events'); +const path = require('path'); +const { readJsonFile, writeJsonFile } = require('./fs-helpers'); + +/** + * Default policy values applied when a new policy is created. + * @readonly + */ +const DEFAULT_POLICY = { + enabled: true, + maxRetries: 3, + retryIntervalMs: 5000, + windowMinutes: 10, + currentRetries: 0, + lastRestartAt: null, + cooldownUntil: null, +}; + +/** + * Manages automatic container restart policies and execution. + * + * @extends EventEmitter + * + * @fires AutoRestartManager#auto-restart-attempt + * @fires AutoRestartManager#auto-restart-success + * @fires AutoRestartManager#auto-restart-failed + * @fires AutoRestartManager#auto-restart-max-reached + */ +class AutoRestartManager extends EventEmitter { + /** + * @param {Object} ctx - Shared application context + * @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode }) + * @param {Object} ctx.healthChecker - HealthChecker singleton + * @param {Object} ctx.notification - NotificationManager instance + * @param {Object} ctx.log - Logger instance + * @param {Function} ctx.logError - Error logging function + * @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir) + */ + constructor(ctx) { + super(); + this.ctx = ctx; + this.log = ctx.log || console; + this.logError = ctx.logError || ((_ctx, err) => console.error(err)); + this.docker = ctx.docker; + this.healthChecker = ctx.healthChecker; + this.notification = ctx.notification; + + /** @type {Map} serviceId -> policy */ + this.policies = new Map(); + + /** Path to the JSON file that persists policies */ + this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json'); + + /** Track previous health status per service for transition detection */ + this._previousHealth = new Map(); + + /** Bound handlers so we can remove them on stop() */ + this._onStatusCheck = this._handleStatusCheck.bind(this); + this._started = false; + } + + // ─── Lifecycle ──────────────────────────────────────────────────────── + + /** + * Load persisted policies, then wire into HealthChecker events. + * @returns {Promise} + */ + async start() { + if (this._started) return; + + // Load persisted policies from disk + try { + const data = await readJsonFile(this.policiesFile, {}); + for (const [serviceId, policy] of Object.entries(data)) { + this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy }); + } + this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size }); + } catch (err) { + this.log.error('auto-restart', 'Failed to load policies', { error: err.message }); + } + + // Listen to health checker status transitions + if (this.healthChecker) { + this.healthChecker.on('status-check', this._onStatusCheck); + } + + this._started = true; + this.log.info('auto-restart', 'Manager started'); + } + + /** + * Remove event listeners and stop processing health events. + */ + stop() { + if (!this._started) return; + + if (this.healthChecker) { + this.healthChecker.removeListener('status-check', this._onStatusCheck); + } + + this._started = false; + this.log.info('auto-restart', 'Manager stopped'); + } + + // ─── Policy CRUD ───────────────────────────────────────────────────── + + /** + * Create or update a restart policy for a service. + * + * @param {string} serviceId - Unique service identifier + * @param {Object} policy - Partial policy fields to merge + * @param {boolean} [policy.enabled=true] + * @param {number} [policy.maxRetries=3] + * @param {number} [policy.retryIntervalMs=5000] + * @param {number} [policy.windowMinutes=10] + * @returns {Promise} The resulting policy + * @throws {Error} If serviceId is invalid + */ + async setPolicy(serviceId, policy) { + if (!serviceId || typeof serviceId !== 'string') { + throw new Error('serviceId is required'); + } + + const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId }; + + const merged = { + ...existing, + ...policy, + serviceId, + // Never allow caller to override runtime counters directly + currentRetries: existing.currentRetries || 0, + lastRestartAt: existing.lastRestartAt, + cooldownUntil: existing.cooldownUntil, + }; + + this.policies.set(serviceId, merged); + await this._savePolicies(); + + this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled }); + return { ...merged }; + } + + /** + * Retrieve the policy for a service. + * + * @param {string} serviceId + * @returns {Object|null} Policy object or null if none exists + */ + getPolicy(serviceId) { + const policy = this.policies.get(serviceId); + return policy ? { ...policy } : null; + } + + /** + * Return all policies as an array. + * @returns {Object[]} + */ + listPolicies() { + return Array.from(this.policies.values()).map(p => ({ ...p })); + } + + /** + * Remove a service's restart policy. + * + * @param {string} serviceId + * @returns {Promise} true if a policy was removed + */ + async removePolicy(serviceId) { + if (!this.policies.has(serviceId)) return false; + + this.policies.delete(serviceId); + await this._savePolicies(); + + this.log.info('auto-restart', 'Policy removed', { serviceId }); + return true; + } + + // ─── Core Restart Logic ────────────────────────────────────────────── + + /** + * Called when a container is detected as down. + * + * Checks policy, cooldown, and retry count, then either attempts a + * Docker restart or notifies that max retries were exceeded. + * + * @param {string} serviceId - Service identifier + * @param {string} containerId - Docker container ID to restart + * @returns {Promise} Result of the operation + */ + async handleContainerDown(serviceId, containerId) { + const policy = this.policies.get(serviceId); + if (!policy) { + return { action: 'ignored', reason: 'no-policy' }; + } + + if (!policy.enabled) { + return { action: 'ignored', reason: 'disabled' }; + } + + // Check cooldown window + const now = Date.now(); + if (policy.cooldownUntil && now < policy.cooldownUntil) { + this.log.info('auto-restart', 'Skipping — cooldown active', { + serviceId, + cooldownUntil: new Date(policy.cooldownUntil).toISOString(), + }); + return { action: 'skipped', reason: 'cooldown' }; + } + + // Max retries exceeded — notify and enter cooldown + if (policy.currentRetries >= policy.maxRetries) { + const cooldownMs = policy.windowMinutes * 60 * 1000; + policy.cooldownUntil = now + cooldownMs; + policy.currentRetries = 0; // Reset so next window can try again + await this._savePolicies(); + + const eventData = { + serviceId, + containerId, + maxRetries: policy.maxRetries, + cooldownUntil: policy.cooldownUntil, + timestamp: new Date().toISOString(), + }; + + /** + * @event AutoRestartManager#auto-restart-max-reached + * @type {Object} + */ + this.emit('auto-restart-max-reached', eventData); + + // Send notification + try { + await this._notify('auto-restart', { + containerName: serviceId, + message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`, + ...eventData, + }); + } catch (notifErr) { + this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); + } + + return { action: 'max-reached', ...eventData }; + } + + // Wait for the configured retry interval before attempting + if (policy.retryIntervalMs > 0 && policy.lastRestartAt) { + const elapsed = now - new Date(policy.lastRestartAt).getTime(); + if (elapsed < policy.retryIntervalMs) { + const waitMs = policy.retryIntervalMs - elapsed; + this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs }); + await new Promise(resolve => setTimeout(resolve, waitMs)); + } + } + + // Attempt restart + policy.currentRetries += 1; + const attemptNum = policy.currentRetries; + const maxRetries = policy.maxRetries; + + /** + * @event AutoRestartManager#auto-restart-attempt + * @type {Object} + */ + this.emit('auto-restart-attempt', { + serviceId, + containerId, + attempt: attemptNum, + maxRetries, + timestamp: new Date().toISOString(), + }); + + try { + if (!this.docker?.client) { + throw new Error('Docker client not available'); + } + + const container = this.docker.client.getContainer(containerId); + await container.start(); + + policy.lastRestartAt = new Date().toISOString(); + await this._savePolicies(); + + const successData = { + serviceId, + containerId, + attempt: attemptNum, + maxRetries, + timestamp: new Date().toISOString(), + }; + + /** + * @event AutoRestartManager#auto-restart-success + * @type {Object} + */ + this.emit('auto-restart-success', successData); + + // Notify + try { + await this._notify('auto-restart', { + containerName: serviceId, + message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`, + ...successData, + }); + } catch (notifErr) { + this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); + } + + this.log.info('auto-restart', 'Container restarted', { + serviceId, + attempt: attemptNum, + maxRetries, + }); + + return { action: 'restarted', ...successData }; + } catch (restartErr) { + policy.lastRestartAt = new Date().toISOString(); + await this._savePolicies(); + + const failData = { + serviceId, + containerId, + attempt: attemptNum, + maxRetries, + error: restartErr.message, + timestamp: new Date().toISOString(), + }; + + /** + * @event AutoRestartManager#auto-restart-failed + * @type {Object} + */ + this.emit('auto-restart-failed', failData); + + // Notify + try { + await this._notify('auto-restart', { + containerName: serviceId, + message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`, + ...failData, + }); + } catch (notifErr) { + this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); + } + + this.log.error('auto-restart', 'Restart failed', { + serviceId, + attempt: attemptNum, + error: restartErr.message, + }); + + return { action: 'failed', ...failData }; + } + } + + /** + * Called when a container recovers to healthy state. + * Resets the retry counter for the associated service. + * + * @param {string} serviceId + * @returns {Promise} + */ + async handleContainerUp(serviceId) { + const policy = this.policies.get(serviceId); + if (!policy) return; + + if (policy.currentRetries > 0) { + policy.currentRetries = 0; + policy.cooldownUntil = null; + await this._savePolicies(); + + this.log.info('auto-restart', 'Retries reset after recovery', { serviceId }); + } + } + + // ─── Health Event Bridge ───────────────────────────────────────────── + + /** + * Internal handler for HealthChecker `status-check` events. + * Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services. + * + * @param {Object} status - HealthChecker status object + * @param {string} status.serviceId + * @param {string} status.status - "up" or "down" + * @private + */ + async _handleStatusCheck(status) { + const { serviceId, status: currentStatus } = status; + if (!serviceId) return; + + // Only process services that have a restart policy + if (!this.policies.has(serviceId)) return; + + const previousStatus = this._previousHealth.get(serviceId); + this._previousHealth.set(serviceId, currentStatus); + + // Transition: healthy → unhealthy + if (previousStatus === 'up' && currentStatus === 'down') { + // Find the containerId from the health checker config or status details + const containerId = this._resolveContainerId(serviceId, status); + if (containerId) { + try { + await this.handleContainerDown(serviceId, containerId); + } catch (err) { + this.logError('auto-restart-health-bridge', err); + } + } + } + + // Transition: unhealthy → healthy (recovery) + if (previousStatus === 'down' && currentStatus === 'up') { + try { + await this.handleContainerUp(serviceId); + } catch (err) { + this.logError('auto-restart-health-bridge', err); + } + } + } + + /** + * Attempt to find the containerId for a service from various sources. + * + * @param {string} serviceId + * @param {Object} status - The status-check event data + * @returns {string|null} + * @private + */ + _resolveContainerId(serviceId, status) { + // Check if it's in the status details (some health checks embed it) + if (status.details?.containerId) return status.details.containerId; + + // Look in the health checker config + const hcService = this.healthChecker?.config?.services?.[serviceId]; + if (hcService?.containerId) return hcService.containerId; + + // Try to look it up from the services state manager + try { + const servicesStateManager = this.ctx.servicesStateManager; + if (servicesStateManager) { + const readResult = servicesStateManager.read(); + if (readResult && typeof readResult.then === 'function') { + // It returns a promise — fire-and-forget lookup + readResult.then(list => { + const found = (list || []).find(s => s.id === serviceId); + return found?.containerId || null; + }).catch(() => null); + } else { + const found = (readResult || []).find(s => s.id === serviceId); + if (found?.containerId) return found.containerId; + } + } + } catch (_) { /* best effort */ } + + return null; + } + + // ─── Persistence ───────────────────────────────────────────────────── + + /** + * Persist current policies to disk. + * @returns {Promise} + * @private + */ + async _savePolicies() { + try { + const obj = {}; + for (const [serviceId, policy] of this.policies.entries()) { + obj[serviceId] = { ...policy }; + } + await writeJsonFile(this.policiesFile, obj); + } catch (err) { + this.log.error('auto-restart', 'Failed to save policies', { error: err.message }); + } + } + + // ─── Helpers ───────────────────────────────────────────────────────── + + /** + * Send a notification via the notification manager. + * + * @param {string} event - Event type (e.g. 'auto-restart') + * @param {Object} data - Notification payload + * @returns {Promise} + * @private + */ + async _notify(event, data) { + if (this.notification?.send) { + return this.notification.send(event, data); + } + return { success: false, reason: 'no-notification-manager' }; + } +} + +module.exports = { AutoRestartManager, DEFAULT_POLICY }; diff --git a/dashcaddy-api/config-drift-detector.js b/dashcaddy-api/config-drift-detector.js new file mode 100644 index 0000000..dbb677b --- /dev/null +++ b/dashcaddy-api/config-drift-detector.js @@ -0,0 +1,376 @@ +/** + * Config Drift Detector - Compares services.json with live Docker state + * + * Detects discrepancies between the configured service list and what is + * actually running in Docker, including missing containers, unknown + * containers, port mismatches, state mismatches, and stale records. + * + * @module config-drift-detector + */ + +const EventEmitter = require('events'); + +/** + * @typedef {Object} DriftReport + * @property {string} checkedAt - ISO timestamp of the check + * @property {Object[]} missingContainers - Services with containerId but container absent in Docker + * @property {Object[]} unknownContainers - Running Docker containers with sami.managed label but not in services.json + * @property {Object[]} portMismatch - Service port != container mapped port + * @property {Object[]} stateMismatch - Service expected up but container stopped/absent + * @property {Object[]} staleRecords - Services with containerId pointing to removed containers + * @property {boolean} hasDrift - Whether any drift category is non-empty + */ + +/** + * Detects and reports configuration drift between services.json and Docker. + * + * @extends EventEmitter + * + * @fires ConfigDriftDetector#drift-detected + */ +class ConfigDriftDetector extends EventEmitter { + /** + * @param {Object} ctx - Shared application context + * @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode }) + * @param {Object} ctx.servicesStateManager - StateManager for services.json + * @param {Object} ctx.notification - NotificationManager instance + * @param {Object} ctx.log - Logger instance + * @param {Function} ctx.logError - Error logging function + */ + constructor(ctx) { + super(); + this.ctx = ctx; + this.log = ctx.log || console; + this.logError = ctx.logError || ((_c, err) => console.error(err)); + this.docker = ctx.docker; + this.servicesStateManager = ctx.servicesStateManager; + this.notification = ctx.notification; + + /** @type {DriftReport|null} Cached report from last detection */ + this.lastReport = null; + + /** @type {NodeJS.Timeout|null} Polling timer reference */ + this._pollTimer = null; + + /** Whether polling is currently active */ + this._polling = false; + } + + // ─── Detection ─────────────────────────────────────────────────────── + + /** + * Run a full drift detection and return the report. + * + * Reads services from servicesStateManager and live containers from Docker, + * then compares them across five drift categories. + * + * @returns {Promise} + */ + async detect() { + const checkedAt = new Date().toISOString(); + + // Gather configured services + let services = []; + try { + const data = await this.servicesStateManager.read(); + services = Array.isArray(data) ? data : (data.services || []); + } catch (err) { + this.log.error('drift', 'Failed to read services', { error: err.message }); + } + + // Gather live Docker containers + let containers = []; + try { + containers = await this.docker.client.listContainers({ all: true }); + } catch (err) { + this.log.error('drift', 'Failed to list containers', { error: err.message }); + } + + // Build lookup maps + const containerById = new Map(); // containerId (short or long) → container info + const containerByName = new Map(); // container name → container info + + for (const c of containers) { + // Store by full ID + containerById.set(c.Id, c); + // Store by short ID (first 12 chars) + if (c.Id && c.Id.length >= 12) { + containerById.set(c.Id.substring(0, 12), c); + } + // Store by name (strip leading /) + for (const name of (c.Names || [])) { + containerByName.set(name.replace(/^\//, ''), c); + } + } + + // Build set of service containerIds for reverse lookup + const serviceContainerIds = new Set(); + const serviceByContainerId = new Map(); + + for (const svc of services) { + if (svc.containerId) { + serviceContainerIds.add(svc.containerId); + // Index by both full and short ID + serviceByContainerId.set(svc.containerId, svc); + if (svc.containerId.length >= 12) { + serviceByContainerId.set(svc.containerId.substring(0, 12), svc); + } + } + } + + const missingContainers = []; + const portMismatch = []; + const stateMismatch = []; + const staleRecords = []; + + for (const svc of services) { + if (!svc.containerId) continue; + + // Look up the container + const container = containerById.get(svc.containerId) + || containerById.get(svc.containerId.substring(0, 12)); + + if (!container) { + // Container ID referenced but not found in Docker at all + staleRecords.push({ + serviceId: svc.id, + name: svc.name, + containerId: svc.containerId, + reason: 'Container not found in Docker', + }); + continue; + } + + // Missing container — service expects it but it's not running + if (container.State !== 'running') { + missingContainers.push({ + serviceId: svc.id, + name: svc.name, + containerId: svc.containerId, + containerState: container.State, + containerStatus: container.Status, + }); + + // Also a state mismatch if the service is expected to be up + stateMismatch.push({ + serviceId: svc.id, + name: svc.name, + expectedState: 'running', + actualState: container.State, + containerId: svc.containerId, + }); + } + + // Port mismatch detection + if (svc.port && container.State === 'running') { + const actualPorts = this._extractContainerPorts(container); + if (actualPorts.length > 0 && !actualPorts.includes(svc.port)) { + portMismatch.push({ + serviceId: svc.id, + name: svc.name, + configuredPort: svc.port, + actualPorts, + containerId: svc.containerId, + }); + } + } + } + + // Unknown managed containers: Docker containers with sami.managed label + // that are NOT in services.json + const unknownContainers = []; + for (const c of containers) { + const isManaged = c.Labels && c.Labels['sami.managed'] === 'true'; + if (!isManaged) continue; + + const isInServices = serviceByContainerId.has(c.Id) + || serviceByContainerId.has(c.Id.substring(0, 12)); + + if (!isInServices) { + unknownContainers.push({ + containerId: c.Id, + name: (c.Names && c.Names[0] || '').replace(/^\//, ''), + image: c.Image, + state: c.State, + status: c.Status, + app: c.Labels?.['sami.app'] || null, + subdomain: c.Labels?.['sami.subdomain'] || null, + }); + } + } + + const report = { + checkedAt, + missingContainers, + unknownContainers, + portMismatch, + stateMismatch, + staleRecords, + hasDrift: missingContainers.length > 0 + || unknownContainers.length > 0 + || portMismatch.length > 0 + || stateMismatch.length > 0 + || staleRecords.length > 0, + }; + + // Cache for quick API access + this.lastReport = report; + + // Emit and notify if drift detected + if (report.hasDrift) { + /** + * @event ConfigDriftDetector#drift-detected + * @type {DriftReport} + */ + this.emit('drift-detected', report); + + try { + await this._sendDriftNotification(report); + } catch (notifErr) { + this.log.error('drift', 'Failed to send drift notification', { + error: notifErr.message, + }); + } + } + + this.log.info('drift', 'Detection complete', { + hasDrift: report.hasDrift, + missing: report.missingContainers.length, + unknown: report.unknownContainers.length, + portMismatch: report.portMismatch.length, + stateMismatch: report.stateMismatch.length, + stale: report.staleRecords.length, + }); + + return report; + } + + // ─── Auto-fix ──────────────────────────────────────────────────────── + + /** + * Attempt to auto-fix drift: + * - Remove stale records (services referencing removed containers) + * - Flag unknown containers for review + * + * @returns {Promise<{ staleRemoved: number, unknownFlagged: number }>} + */ + async autoFix() { + const report = await this.detect(); + let staleRemoved = 0; + + // Remove stale records from services.json + if (report.staleRecords.length > 0) { + const staleIds = new Set(report.staleRecords.map(r => r.serviceId)); + await this.servicesStateManager.update(services => { + const before = services.length; + const cleaned = services.filter(s => !staleIds.has(s.id)); + staleRemoved = before - cleaned.length; + return cleaned; + }); + } + + const unknownFlagged = report.unknownContainers.length; + + this.log.info('drift', 'Auto-fix applied', { staleRemoved, unknownFlagged }); + + return { staleRemoved, unknownFlagged }; + } + + // ─── Polling ───────────────────────────────────────────────────────── + + /** + * Start periodic drift detection. + * + * @param {number} [intervalMs=300000] - Polling interval in milliseconds (default 5 min) + */ + startPolling(intervalMs = 300000) { + this.stopPolling(); + + this._polling = true; + this._pollTimer = setInterval(async () => { + try { + await this.detect(); + } catch (err) { + this.logError('drift-poll', err); + } + }, intervalMs); + + this.log.info('drift', 'Polling started', { intervalMs }); + } + + /** + * Stop periodic drift detection. + */ + stopPolling() { + if (this._pollTimer) { + clearInterval(this._pollTimer); + this._pollTimer = null; + } + this._polling = false; + this.log.info('drift', 'Polling stopped'); + } + + /** + * Whether polling is currently active. + * @returns {boolean} + */ + isPolling() { + return this._polling; + } + + // ─── Helpers ───────────────────────────────────────────────────────── + + /** + * Extract mapped host ports from a Docker container info object. + * + * @param {Object} container - Dockerode container info + * @returns {number[]} Array of host port numbers + * @private + */ + _extractContainerPorts(container) { + const ports = []; + if (!container.Ports) return ports; + + for (const p of container.Ports) { + if (p.PublicPort) { + ports.push(p.PublicPort); + } + } + + return ports; + } + + /** + * Send a notification about detected drift. + * + * @param {DriftReport} report + * @returns {Promise} + * @private + */ + async _sendDriftNotification(report) { + if (!this.notification?.send) { + return { success: false, reason: 'no-notification-manager' }; + } + + const parts = []; + if (report.missingContainers.length > 0) { + parts.push(`Missing containers: ${report.missingContainers.map(c => c.name).join(', ')}`); + } + if (report.unknownContainers.length > 0) { + parts.push(`Unknown managed containers: ${report.unknownContainers.map(c => c.name).join(', ')}`); + } + if (report.portMismatch.length > 0) { + parts.push(`Port mismatches: ${report.portMismatch.map(c => c.name).join(', ')}`); + } + if (report.staleRecords.length > 0) { + parts.push(`Stale records: ${report.staleRecords.map(c => c.name).join(', ')}`); + } + + return this.notification.send('drift-detected', { + text: `⚠️ Configuration drift detected:\n${parts.join('\n')}`, + report, + }); + } +} + +module.exports = { ConfigDriftDetector }; diff --git a/dashcaddy-api/dependency-manager.js b/dashcaddy-api/dependency-manager.js new file mode 100644 index 0000000..6cd19bc --- /dev/null +++ b/dashcaddy-api/dependency-manager.js @@ -0,0 +1,605 @@ +/** + * Dependency Manager - Service dependency tracking with ordered restart chains + * + * Manages directed acyclic graph (DAG) of service dependencies. Services can + * declare which other services they depend on, and this manager provides: + * - Full dependency graph inspection + * - Topological ordering for safe restart chains + * - Circular dependency detection + * - Health-aware restart with per-service polling + * + * Dependencies are stored directly on service objects in services.json: + * { id, name, ..., dependsOn: ['service-id-1', 'service-id-2'] } + * + * @module dependency-manager + */ + +const EventEmitter = require('events'); + +/** Maximum seconds to wait for a single container to become healthy after restart */ +const HEALTH_CHECK_TIMEOUT_MS = 30_000; + +/** Interval between container health polls */ +const HEALTH_CHECK_INTERVAL_MS = 1_000; + +/** + * @typedef {Object} ServiceNode + * @property {string} serviceId + * @property {string} name + * @property {string|null} containerId + */ + +/** + * @typedef {Object} DependencyEdge + * @property {string} from - The service that depends + * @property {string} to - The service being depended upon + */ + +/** + * @typedef {Object} DependencyGraph + * @property {ServiceNode[]} nodes + * @property {DependencyEdge[]} edges + */ + +/** + * @typedef {Object} DependencyStatusEntry + * @property {string} serviceId + * @property {string} name + * @property {boolean} isUp + * @property {string} [error] + */ + +/** + * DependencyManager — tracks service dependencies and orchestrates ordered restarts. + * + * Events emitted: + * - `dependency-restart-start` ({ serviceId, chain: string[] }) + * - `dependency-restart-progress` ({ serviceId, currentServiceId, index, total }) + * - `dependency-restart-complete` ({ serviceId, chain: string[], results: Array }) + * - `dependency-restart-failed` ({ serviceId, failedServiceId, error, chain: string[] }) + * + * @extends EventEmitter + */ +class DependencyManager extends EventEmitter { + /** + * @param {Object} ctx - Application context + * @param {Object} ctx.servicesStateManager - StateManager for services.json + * @param {Object} ctx.docker - Docker context ({ client: Dockerode }) + * @param {Object} ctx.notification - NotificationManager instance + * @param {Object} ctx.log - Logger instance + */ + constructor(ctx) { + super(); + /** @private */ + this.ctx = ctx; + /** @private */ + this._servicesStateManager = ctx.servicesStateManager; + /** @private */ + this._docker = ctx.docker; + /** @private */ + this._notification = ctx.notification; + /** @private */ + this._log = ctx.log || console; + } + + // --------------------------------------------------------------------------- + // Core helpers + // --------------------------------------------------------------------------- + + /** + * Load all services from the state manager. + * @private + * @returns {Promise} + */ + async _loadServices() { + const data = await this._servicesStateManager.read(); + return Array.isArray(data) ? data : (data.services || []); + } + + /** + * Find a single service by ID. + * @private + * @param {string} serviceId + * @returns {Promise} + */ + async _findService(serviceId) { + const services = await this._loadServices(); + return services.find(s => s.id === serviceId) || null; + } + + // --------------------------------------------------------------------------- + // Graph queries + // --------------------------------------------------------------------------- + + /** + * Return the full dependency graph for visualisation. + * + * @returns {Promise} + */ + async getDependencyGraph() { + const services = await this._loadServices(); + + const nodes = services.map(s => ({ + serviceId: s.id, + name: s.name, + containerId: s.containerId || null, + })); + + const edges = []; + for (const service of services) { + const deps = service.dependsOn || []; + for (const depId of deps) { + edges.push({ from: service.id, to: depId }); + } + } + + return { nodes, edges }; + } + + /** + * Return the services that depend on the given service (reverse deps). + * + * @param {string} serviceId + * @returns {Promise} Services whose `dependsOn` includes `serviceId`. + */ + async getDependents(serviceId) { + const services = await this._loadServices(); + return services.filter(s => (s.dependsOn || []).includes(serviceId)); + } + + /** + * Return the direct dependencies for a service. + * + * @param {string} serviceId + * @returns {Promise} Services that `serviceId` depends on. + */ + async getDependencies(serviceId) { + const services = await this._loadServices(); + const service = services.find(s => s.id === serviceId); + if (!service) return []; + const depIds = service.dependsOn || []; + return services.filter(s => depIds.includes(s.id)); + } + + // --------------------------------------------------------------------------- + // Topological sort + // --------------------------------------------------------------------------- + + /** + * Build an adjacency list for the current dependency graph. + * Edge direction: service → its dependencies (i.e. what it depends on). + * + * @private + * @param {Object[]} services + * @returns {Map} + */ + _buildAdjacencyList(services) { + const adj = new Map(); + for (const service of services) { + adj.set(service.id, (service.dependsOn || []).slice()); + } + return adj; + } + + /** + * DFS-based topological sort with cycle detection (white/gray/black coloring). + * + * Returns services in restart order: dependencies first, dependents last. + * The target service is included at the end. + * + * @private + * @param {string} serviceId - Target service (will be last in the result). + * @param {Object[]} services - All services. + * @param {Map} adj - Adjacency list (service → deps). + * @returns {string[]} Ordered service IDs for restart. + * @throws {Error} If a circular dependency is detected. + */ + _topologicalSort(serviceId, services, adj) { + // Collect only the reachable sub-graph from serviceId + const visited = new Set(); + const reachable = new Set(); + + const collectReachable = (id) => { + if (reachable.has(id)) return; + reachable.add(id); + for (const dep of (adj.get(id) || [])) { + collectReachable(dep); + } + }; + collectReachable(serviceId); + + // DFS topological sort on the reachable sub-graph + const WHITE = 0, GRAY = 1, BLACK = 2; + const color = new Map(); + for (const id of reachable) color.set(id, WHITE); + + const result = []; + + const dfs = (id) => { + if (color.get(id) === BLACK) return; + if (color.get(id) === GRAY) { + throw new Error(`Circular dependency detected involving service "${id}"`); + } + color.set(id, GRAY); + for (const dep of (adj.get(id) || [])) { + dfs(dep); + } + color.set(id, BLACK); + result.push(id); + }; + + // Visit the target last so it ends up at the end of the result + // Actually, we want deps *first* then the target. + // The DFS naturally puts deps before dependents, so starting from + // serviceId will place it last (which is correct for restart order). + dfs(serviceId); + + return result; + } + + /** + * Get the topologically ordered restart chain for a service. + * + * The returned array lists all services that must be restarted, + * starting with leaf dependencies and ending with the target service. + * + * @param {string} serviceId - The service to build the chain for. + * @returns {Promise} Ordered service IDs. + * @throws {Error} If `serviceId` doesn't exist or a circular dependency is found. + */ + async getOrderedRestartChain(serviceId) { + const services = await this._loadServices(); + const service = services.find(s => s.id === serviceId); + if (!service) { + throw new Error(`Service "${serviceId}" not found`); + } + + const adj = this._buildAdjacencyList(services); + return this._topologicalSort(serviceId, services, adj); + } + + // --------------------------------------------------------------------------- + // Validation + // --------------------------------------------------------------------------- + + /** + * Validate a proposed set of dependencies for a service. + * + * Checks: + * - All referenced service IDs exist. + * - Adding these dependencies would not create a circular dependency. + * - A service cannot depend on itself. + * + * @param {string} serviceId - The service to set dependencies on. + * @param {string[]} dependsOn - Proposed dependency IDs. + * @returns {Promise<{ valid: boolean, errors: string[] }>} + */ + async validateDependencies(serviceId, dependsOn) { + const errors = []; + + if (!Array.isArray(dependsOn)) { + return { valid: false, errors: ['dependsOn must be an array'] }; + } + + const services = await this._loadServices(); + const allIds = new Set(services.map(s => s.id)); + + // Service must exist + if (!allIds.has(serviceId)) { + return { valid: false, errors: [`Service "${serviceId}" not found`] }; + } + + // Self-dependency + if (dependsOn.includes(serviceId)) { + errors.push(`Service "${serviceId}" cannot depend on itself`); + } + + // Existence check + for (const depId of dependsOn) { + if (!allIds.has(depId)) { + errors.push(`Dependency service "${depId}" does not exist`); + } + } + + if (errors.length > 0) { + return { valid: false, errors }; + } + + // Circular dependency check: temporarily set the proposed dependsOn + // and attempt a topological sort. + const tempServices = services.map(s => { + if (s.id === serviceId) { + return { ...s, dependsOn: dependsOn.slice() }; + } + return { ...s }; + }); + + const adj = this._buildAdjacencyList(tempServices); + + // Check every node for cycles with the new edges + try { + const WHITE = 0, GRAY = 1, BLACK = 2; + const color = new Map(); + for (const s of tempServices) color.set(s.id, WHITE); + + const dfs = (id) => { + if (color.get(id) === BLACK) return; + if (color.get(id) === GRAY) { + throw new Error(`Circular dependency detected involving service "${id}"`); + } + color.set(id, GRAY); + for (const dep of (adj.get(id) || [])) { + dfs(dep); + } + color.set(id, BLACK); + }; + + for (const s of tempServices) { + if (color.get(s.id) === WHITE) { + dfs(s.id); + } + } + } catch (err) { + errors.push(err.message); + } + + return { valid: errors.length === 0, errors }; + } + + // --------------------------------------------------------------------------- + // Health status + // --------------------------------------------------------------------------- + + /** + * Get the current container status for a service and all its transitive dependencies. + * + * @param {string} serviceId + * @returns {Promise} + * @throws {Error} If `serviceId` doesn't exist. + */ + async getDependencyStatus(serviceId) { + const services = await this._loadServices(); + const service = services.find(s => s.id === serviceId); + if (!service) { + throw new Error(`Service "${serviceId}" not found`); + } + + // Collect all transitive dependencies via BFS + const serviceMap = new Map(services.map(s => [s.id, s])); + const visited = new Set(); + const queue = [serviceId]; + const allRelated = []; + + while (queue.length > 0) { + const currentId = queue.shift(); + if (visited.has(currentId)) continue; + visited.add(currentId); + + const svc = serviceMap.get(currentId); + if (!svc) continue; + + allRelated.push(svc); + + for (const depId of (svc.dependsOn || [])) { + if (!visited.has(depId)) { + queue.push(depId); + } + } + } + + // Query container status for each + const results = []; + for (const svc of allRelated) { + const entry = { + serviceId: svc.id, + name: svc.name, + isUp: false, + }; + + if (!svc.containerId) { + entry.error = 'No container associated with this service'; + results.push(entry); + continue; + } + + try { + const container = this._docker.client.getContainer(svc.containerId); + const info = await container.inspect(); + entry.isUp = info.State?.Running === true; + } catch (err) { + entry.error = err.message || 'Unable to inspect container'; + } + + results.push(entry); + } + + return results; + } + + // --------------------------------------------------------------------------- + // Restart with dependencies + // --------------------------------------------------------------------------- + + /** + * Wait for a container to report as running after a restart. + * + * @private + * @param {string} containerId + * @param {number} [timeoutMs=30000] + * @returns {Promise} `true` if healthy, `false` if timed out. + */ + async _waitForContainerHealthy(containerId, timeoutMs = HEALTH_CHECK_TIMEOUT_MS) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + try { + const container = this._docker.client.getContainer(containerId); + const info = await container.inspect(); + if (info.State?.Running === true) { + return true; + } + } catch { + // Container might not be inspectable during restart — keep polling + } + await new Promise(r => setTimeout(r, HEALTH_CHECK_INTERVAL_MS)); + } + return false; + } + + /** + * Restart a service and all its dependencies in topological order. + * + * Emits progress events and sends a notification on completion/failure. + * This method is designed to be called from the route handler and + * **does not throw** — errors are reported via events and notifications. + * + * @param {string} serviceId - Target service to restart (with deps). + * @returns {Promise<{ success: boolean, chain: string[], results: Array }>} + */ + async restartWithDependencies(serviceId) { + const service = await this._findService(serviceId); + if (!service) { + const err = new Error(`Service "${serviceId}" not found`); + this.emit('dependency-restart-failed', { + serviceId, + failedServiceId: serviceId, + error: err.message, + chain: [], + }); + throw err; + } + + let chain; + try { + chain = await this.getOrderedRestartChain(serviceId); + } catch (err) { + this.emit('dependency-restart-failed', { + serviceId, + failedServiceId: serviceId, + error: err.message, + chain: [], + }); + throw err; + } + + const services = await this._loadServices(); + const serviceMap = new Map(services.map(s => [s.id, s])); + + this._log.info('dependency', 'Starting dependency restart chain', { + serviceId, + chain, + }); + + this.emit('dependency-restart-start', { serviceId, chain }); + + const results = []; + const total = chain.length; + + for (let i = 0; i < total; i++) { + const currentId = chain[i]; + const svc = serviceMap.get(currentId); + + this.emit('dependency-restart-progress', { + serviceId, + currentServiceId: currentId, + index: i, + total, + }); + + if (!svc || !svc.containerId) { + const msg = !svc + ? `Service "${currentId}" not found in state` + : `Service "${currentId}" has no container — skipping restart`; + this._log.warn('dependency', msg); + results.push({ serviceId: currentId, restarted: false, skipped: true, reason: msg }); + continue; + } + + try { + const container = this._docker.client.getContainer(svc.containerId); + this._log.info('dependency', `Restarting container for service "${currentId}"`, { + containerId: svc.containerId, + }); + await container.restart(); + + // Wait for it to come back up + const healthy = await this._waitForContainerHealthy(svc.containerId); + if (!healthy) { + const msg = `Container for service "${currentId}" did not become healthy within ${HEALTH_CHECK_TIMEOUT_MS / 1000}s`; + this._log.warn('dependency', msg); + results.push({ serviceId: currentId, restarted: true, healthy: false, error: msg }); + + // Abort chain — dependency didn't come back + this.emit('dependency-restart-failed', { + serviceId, + failedServiceId: currentId, + error: msg, + chain, + }); + await this._notifyRestartResult(serviceId, false, chain, results, currentId); + return { success: false, chain, results }; + } + + this._log.info('dependency', `Service "${currentId}" is healthy after restart`); + results.push({ serviceId: currentId, restarted: true, healthy: true }); + } catch (err) { + const msg = err.message || 'Unknown error during restart'; + this._log.error('dependency', `Failed to restart service "${currentId}"`, { + error: msg, + }); + results.push({ serviceId: currentId, restarted: false, error: msg }); + + this.emit('dependency-restart-failed', { + serviceId, + failedServiceId: currentId, + error: msg, + chain, + }); + await this._notifyRestartResult(serviceId, false, chain, results, currentId); + return { success: false, chain, results }; + } + } + + this.emit('dependency-restart-complete', { serviceId, chain, results }); + await this._notifyRestartResult(serviceId, true, chain, results); + return { success: true, chain, results }; + } + + /** + * Send a notification about the restart result. + * + * @private + * @param {string} serviceId + * @param {boolean} success + * @param {string[]} chain + * @param {Array} results + * @param {string} [failedServiceId] + */ + async _notifyRestartResult(serviceId, success, chain, results, failedServiceId) { + if (!this._notification) return; + + try { + if (success) { + await this._notification.send('dependency-restart-complete', { + text: `✅ Dependency restart chain completed for "${serviceId}". Restarted: ${chain.join(' → ')}`, + serviceId, + chain, + results, + }); + } else { + await this._notification.send('dependency-restart-failed', { + text: `❌ Dependency restart chain failed for "${serviceId}" at "${failedServiceId}". Chain: ${chain.join(' → ')}`, + serviceId, + failedServiceId, + chain, + results, + }); + } + } catch (err) { + this._log.error('dependency', 'Failed to send restart notification', { + error: err.message, + }); + } + } +} + +module.exports = DependencyManager; diff --git a/dashcaddy-api/dns-propagation.js b/dashcaddy-api/dns-propagation.js new file mode 100644 index 0000000..f00416b --- /dev/null +++ b/dashcaddy-api/dns-propagation.js @@ -0,0 +1,273 @@ +/** + * DNS Propagation Checker + * Verifies DNS record propagation by querying multiple resolvers. + * Runs as background jobs with configurable timeout and interval. + * + * @module dns-propagation + */ + +const dns = require('dns').promises; +const EventEmitter = require('events'); + +/** Default verification options */ +const DEFAULT_OPTIONS = { + timeout: 300000, // 5 minutes + interval: 10000, // 10 seconds + resolvers: ['1.1.1.1', '8.8.8.8', '9.9.9.9'] +}; + +/** Maximum age for stored verification results (1 hour) */ +const MAX_RESULT_AGE_MS = 3600000; + +class DNSPropagationChecker extends EventEmitter { + /** + * Create a DNSPropagationChecker instance. + * @param {Object} ctx - Shared application context + * @param {Object} ctx.notification - NotificationManager instance + * @param {Object} ctx.log - Logger instance + */ + constructor(ctx) { + super(); + this.ctx = ctx; + this.log = ctx.log || console; + + /** @type {Map} domain → verification status */ + this.verifications = new Map(); + } + + /** + * Verify that a DNS record has propagated by querying multiple resolvers. + * Retries every `interval` ms until `timeout` is reached. + * + * @param {string} domain - The domain to check (e.g., 'test.sami') + * @param {string} expectedIp - The expected IP address + * @param {Object} [options={}] - Verification options + * @param {number} [options.timeout=300000] - Maximum time to wait (ms) + * @param {number} [options.interval=10000] - Time between retries (ms) + * @param {string[]} [options.resolvers] - DNS resolvers to query + * @returns {Promise} Verification result + */ + async verifyRecord(domain, expectedIp, options = {}) { + const startTime = Date.now(); + const { + timeout = DEFAULT_OPTIONS.timeout, + interval = DEFAULT_OPTIONS.interval, + resolvers = DEFAULT_OPTIONS.resolvers + } = options; + + const allResults = []; + let propagated = false; + + while (Date.now() - startTime < timeout) { + const roundResults = []; + + for (const resolver of resolvers) { + const checkStart = Date.now(); + try { + // Use dns.resolve4 with a custom resolver + const resolverInstance = new dns.Resolver(); + resolverInstance.setServers([resolver]); + resolverInstance.setTimeout(5000); + + const addresses = await resolverInstance.resolve4(domain); + const matched = addresses.includes(expectedIp); + + const result = { + resolver, + ips: addresses, + matched, + checkedAt: new Date().toISOString(), + responseTime: Date.now() - checkStart + }; + + roundResults.push(result); + + if (matched) { + propagated = true; + } + } catch (err) { + roundResults.push({ + resolver, + ips: [], + matched: false, + checkedAt: new Date().toISOString(), + error: err.code || err.message, + responseTime: Date.now() - checkStart + }); + } + } + + allResults.push(...roundResults); + + // Emit progress event + this.emit('propagation-check', { + domain, + expectedIp, + roundResults, + elapsed: Date.now() - startTime, + propagated + }); + + if (propagated) { + break; + } + + // Wait before next attempt + await new Promise(resolve => setTimeout(resolve, interval)); + } + + const totalTime = Date.now() - startTime; + + return { + domain, + expectedIp, + propagated, + results: allResults, + totalTime, + checkedAt: new Date().toISOString() + }; + } + + /** + * Start a background DNS propagation verification. + * Does not block — returns immediately with the job reference. + * + * @param {string} domain - The domain to verify + * @param {string} expectedIp - The expected IP address + * @param {Object} [options={}] - Verification options + * @returns {Object} Job status object + */ + startVerification(domain, expectedIp, options = {}) { + // If there's already a running verification for this domain, return it + const existing = this.verifications.get(domain); + if (existing && existing.status === 'running') { + return existing; + } + + const job = { + domain, + expectedIp, + status: 'running', + startedAt: new Date().toISOString(), + progress: [], + result: null + }; + + this.verifications.set(domain, job); + + // Run verification in background (non-blocking) + this.verifyRecord(domain, expectedIp, options) + .then(result => { + job.status = 'completed'; + job.result = result; + job.completedAt = new Date().toISOString(); + + if (result.propagated) { + this.emit('propagation-complete', result); + + if (this.ctx.notification) { + this.ctx.notification.send('dns-propagation', { + text: `✅ DNS record for ${domain} propagated successfully to ${expectedIp}`, + domain, + expectedIp, + totalTime: result.totalTime + }, 'success').catch(err => { + this.log.error('dns-propagation', 'Failed to send propagation notification', { + error: err.message + }); + }); + } + } else { + this.emit('propagation-timeout', result); + + if (this.ctx.notification) { + this.ctx.notification.send('dns-propagation', { + text: `⏱️ DNS propagation timeout for ${domain} — expected ${expectedIp} not found after ${Math.round(result.totalTime / 1000)}s`, + domain, + expectedIp, + totalTime: result.totalTime + }, 'warning').catch(err => { + this.log.error('dns-propagation', 'Failed to send timeout notification', { + error: err.message + }); + }); + } + } + }) + .catch(err => { + job.status = 'error'; + job.error = err.message; + job.completedAt = new Date().toISOString(); + + this.log.error('dns-propagation', `Verification failed for ${domain}`, { + error: err.message + }); + }); + + return job; + } + + /** + * Get the current verification status for a domain. + * + * @param {string} domain - The domain to look up + * @returns {Object|null} Verification status or null if not found + */ + getVerificationStatus(domain) { + const job = this.verifications.get(domain); + if (!job) return null; + return { + domain: job.domain, + expectedIp: job.expectedIp, + status: job.status, + startedAt: job.startedAt, + completedAt: job.completedAt || null, + result: job.result || null, + error: job.error || null + }; + } + + /** + * Get all recent verifications. + * + * @returns {Object[]} Array of verification statuses + */ + getAllVerifications() { + const results = []; + for (const [domain, job] of this.verifications.entries()) { + results.push({ + domain, + expectedIp: job.expectedIp, + status: job.status, + startedAt: job.startedAt, + completedAt: job.completedAt || null, + propagated: job.result?.propagated || null, + totalTime: job.result?.totalTime || null, + error: job.error || null + }); + } + return results; + } + + /** + * Remove verifications older than 1 hour. + */ + cleanup() { + const now = Date.now(); + for (const [domain, job] of this.verifications.entries()) { + const completedAt = job.completedAt ? new Date(job.completedAt).getTime() : null; + const startedAt = new Date(job.startedAt).getTime(); + + // Clean up completed/error jobs older than 1 hour + // Also clean up stale running jobs that started over 2 hours ago + const age = completedAt ? (now - completedAt) : (now - startedAt); + const maxAge = job.status === 'running' ? MAX_RESULT_AGE_MS * 2 : MAX_RESULT_AGE_MS; + + if (age > maxAge) { + this.verifications.delete(domain); + } + } + } +} + +module.exports = DNSPropagationChecker; diff --git a/dashcaddy-api/package.json b/dashcaddy-api/package.json index 6b955ad..001a88c 100644 --- a/dashcaddy-api/package.json +++ b/dashcaddy-api/package.json @@ -1,6 +1,6 @@ { "name": "dashcaddy-api", - "version": "1.8.0", + "version": "1.9.0", "description": "DashCaddy API server - Dashboard backend for Docker, Caddy & DNS management", "main": "server.js", "scripts": { diff --git a/dashcaddy-api/routes/auto-restart.js b/dashcaddy-api/routes/auto-restart.js new file mode 100644 index 0000000..0357548 --- /dev/null +++ b/dashcaddy-api/routes/auto-restart.js @@ -0,0 +1,164 @@ +/** + * Auto-Restart Policy Routes + * + * CRUD endpoints for per-container auto-restart policies. + * Also provides a dry-run test endpoint. + * + * @module routes/auto-restart + */ + +const express = require('express'); +const { success } = require('../response-helpers'); +const { ValidationError, NotFoundError } = require('../errors'); + +/** + * Auto-restart route factory + * + * @param {Object} deps - Explicit dependencies + * @param {Object} deps.autoRestartManager - AutoRestartManager instance + * @param {Function} deps.asyncHandler - Async route handler wrapper + * @param {Function} deps.logError - Error logging function + * @returns {express.Router} + */ +module.exports = function ({ autoRestartManager, asyncHandler, logError }) { + const router = express.Router(); + + /** + * GET /auto-restart/policies + * List all configured auto-restart policies. + */ + router.get('/policies', asyncHandler(async (_req, res) => { + const policies = autoRestartManager.listPolicies(); + success(res, { policies }); + }, 'auto-restart-list')); + + /** + * GET /auto-restart/policies/:serviceId + * Get the restart policy for a single service. + */ + router.get('/policies/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) { + throw new ValidationError('Invalid service ID format'); + } + + const policy = autoRestartManager.getPolicy(serviceId); + if (!policy) { + throw new NotFoundError(`Auto-restart policy for "${serviceId}"`); + } + + success(res, { policy }); + }, 'auto-restart-get')); + + /** + * POST /auto-restart/policies/:serviceId + * Create or update a restart policy. + * + * Body: { enabled, maxRetries, retryIntervalMs, windowMinutes } + */ + router.post('/policies/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) { + throw new ValidationError('Invalid service ID format'); + } + + const { enabled, maxRetries, retryIntervalMs, windowMinutes } = req.body; + + // Validate inputs + if (enabled !== undefined && typeof enabled !== 'boolean') { + throw new ValidationError('enabled must be a boolean'); + } + if (maxRetries !== undefined) { + if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 100) { + throw new ValidationError('maxRetries must be an integer between 0 and 100'); + } + } + if (retryIntervalMs !== undefined) { + if (!Number.isInteger(retryIntervalMs) || retryIntervalMs < 0 || retryIntervalMs > 3600000) { + throw new ValidationError('retryIntervalMs must be an integer between 0 and 3600000'); + } + } + if (windowMinutes !== undefined) { + if (!Number.isInteger(windowMinutes) || windowMinutes < 0 || windowMinutes > 1440) { + throw new ValidationError('windowMinutes must be an integer between 0 and 1440'); + } + } + + const policy = await autoRestartManager.setPolicy(serviceId, { + ...(enabled !== undefined && { enabled }), + ...(maxRetries !== undefined && { maxRetries }), + ...(retryIntervalMs !== undefined && { retryIntervalMs }), + ...(windowMinutes !== undefined && { windowMinutes }), + }); + + success(res, { policy, message: `Policy ${serviceId} saved` }); + }, 'auto-restart-set')); + + /** + * DELETE /auto-restart/policies/:serviceId + * Remove a restart policy. + */ + router.delete('/policies/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) { + throw new ValidationError('Invalid service ID format'); + } + + const removed = await autoRestartManager.removePolicy(serviceId); + if (!removed) { + throw new NotFoundError(`Auto-restart policy for "${serviceId}"`); + } + + success(res, { message: `Policy for "${serviceId}" removed` }); + }, 'auto-restart-delete')); + + /** + * POST /auto-restart/policies/:serviceId/test + * Dry-run: simulate a restart attempt without actually restarting. + * Returns what *would* happen given the current policy state. + */ + router.post('/policies/:serviceId/test', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) { + throw new ValidationError('Invalid service ID format'); + } + + const policy = autoRestartManager.getPolicy(serviceId); + if (!policy) { + throw new NotFoundError(`Auto-restart policy for "${serviceId}"`); + } + + const now = Date.now(); + const inCooldown = policy.cooldownUntil && now < policy.cooldownUntil; + const wouldRetry = !inCooldown && policy.currentRetries < policy.maxRetries; + const nextAttempt = policy.currentRetries + 1; + + success(res, { + dryRun: true, + serviceId, + policy: { + enabled: policy.enabled, + currentRetries: policy.currentRetries, + maxRetries: policy.maxRetries, + cooldownUntil: policy.cooldownUntil, + inCooldown, + }, + wouldRestart: policy.enabled && wouldRetry, + wouldMaxOut: !wouldRetry && !inCooldown, + nextAttempt: wouldRetry ? nextAttempt : null, + message: !policy.enabled + ? 'Policy is disabled — no restart would occur' + : inCooldown + ? `In cooldown until ${new Date(policy.cooldownUntil).toISOString()} — would skip` + : wouldRetry + ? `Would attempt restart ${nextAttempt}/${policy.maxRetries}` + : `Max retries (${policy.maxRetries}) already reached — would enter cooldown`, + }); + }, 'auto-restart-test')); + + return router; +}; diff --git a/dashcaddy-api/routes/config-drift.js b/dashcaddy-api/routes/config-drift.js new file mode 100644 index 0000000..e779004 --- /dev/null +++ b/dashcaddy-api/routes/config-drift.js @@ -0,0 +1,92 @@ +/** + * Config Drift Detection Routes + * + * API endpoints for running drift detection, reading cached reports, + * auto-fixing drift, and controlling periodic polling. + * + * @module routes/config-drift + */ + +const express = require('express'); +const { success } = require('../response-helpers'); +const { ValidationError, NotFoundError } = require('../errors'); + +/** + * Config-drift route factory + * + * @param {Object} deps - Explicit dependencies + * @param {Object} deps.driftDetector - ConfigDriftDetector instance + * @param {Function} deps.asyncHandler - Async route handler wrapper + * @param {Function} deps.logError - Error logging function + * @returns {express.Router} + */ +module.exports = function ({ driftDetector, asyncHandler, logError }) { + const router = express.Router(); + + /** + * GET /config-drift/report + * Run a fresh drift detection and return the full report. + */ + router.get('/report', asyncHandler(async (_req, res) => { + const report = await driftDetector.detect(); + success(res, { report }); + }, 'drift-report')); + + /** + * GET /config-drift/last + * Return the last cached drift report (no re-detection). + */ + router.get('/last', asyncHandler(async (_req, res) => { + if (!driftDetector.lastReport) { + throw new NotFoundError('No cached drift report — run detection first'); + } + + success(res, { report: driftDetector.lastReport }); + }, 'drift-last')); + + /** + * POST /config-drift/fix + * Auto-fix detected drift: remove stale records, flag unknown containers. + */ + router.post('/fix', asyncHandler(async (_req, res) => { + const result = await driftDetector.autoFix(); + success(res, { + message: 'Auto-fix applied', + staleRemoved: result.staleRemoved, + unknownFlagged: result.unknownFlagged, + }); + }, 'drift-fix')); + + /** + * POST /config-drift/polling + * Enable or disable periodic drift detection polling. + * + * Body: { enabled: boolean, intervalMs?: number } + */ + router.post('/polling', asyncHandler(async (req, res) => { + const { enabled, intervalMs } = req.body; + + if (typeof enabled !== 'boolean') { + throw new ValidationError('enabled must be a boolean'); + } + + if (intervalMs !== undefined) { + if (!Number.isInteger(intervalMs) || intervalMs < 10000 || intervalMs > 86400000) { + throw new ValidationError('intervalMs must be an integer between 10000 and 86400000 (10s – 24h)'); + } + } + + if (enabled) { + driftDetector.startPolling(intervalMs || 300000); + success(res, { + message: 'Drift polling enabled', + intervalMs: intervalMs || 300000, + }); + } else { + driftDetector.stopPolling(); + success(res, { message: 'Drift polling disabled' }); + } + }, 'drift-polling')); + + return router; +}; diff --git a/dashcaddy-api/routes/dependencies.js b/dashcaddy-api/routes/dependencies.js new file mode 100644 index 0000000..11629dd --- /dev/null +++ b/dashcaddy-api/routes/dependencies.js @@ -0,0 +1,235 @@ +/** + * Dependencies Route — REST API for service dependency tracking + * + * Endpoints: + * GET /dependencies/graph Full dependency graph + * GET /dependencies/validate Validate a proposed dep chain + * GET /dependencies/:serviceId Direct deps for one service + * GET /dependencies/:serviceId/chain Ordered restart chain + * GET /dependencies/:serviceId/status Dependency health status + * POST /dependencies/:serviceId Set dependencies + * DELETE /dependencies/:serviceId Remove all dependencies + * POST /dependencies/:serviceId/restart Restart with dependency chain + * + * @module routes/dependencies + */ + +const express = require('express'); +const { success, error: errorResponse } = require('../response-helpers'); +const { NotFoundError, ValidationError } = require('../errors'); + +/** + * Dependencies route factory + * + * @param {Object} deps - Explicit dependencies + * @param {Object} deps.dependencyManager - DependencyManager instance + * @param {Object} deps.servicesStateManager - State manager for services.json + * @param {Object} deps.docker - Docker client wrapper + * @param {Function} deps.asyncHandler - Async route handler wrapper + * @param {Function} deps.logError - Error logging function + * @param {Function} deps.resyncHealthChecker - Health checker resync function + * @param {Object} deps.log - Logger instance + * @returns {express.Router} + */ +module.exports = function({ + dependencyManager, + servicesStateManager, + docker, + asyncHandler, + logError, + resyncHealthChecker, + log, +}) { + const router = express.Router(); + + // ------------------------------------------------------------------------- + // GET /dependencies/graph — Full dependency graph + // ------------------------------------------------------------------------- + router.get('/graph', asyncHandler(async (req, res) => { + const graph = await dependencyManager.getDependencyGraph(); + success(res, { graph }); + }, 'dep-graph')); + + // ------------------------------------------------------------------------- + // GET /dependencies/validate — Validate a proposed dep chain (query params) + // ------------------------------------------------------------------------- + router.get('/validate', asyncHandler(async (req, res) => { + const { serviceId, dependsOn } = req.query; + + if (!serviceId) { + throw new ValidationError('serviceId query parameter is required'); + } + + // dependsOn may be a comma-separated string or already an array + let parsed; + if (Array.isArray(dependsOn)) { + parsed = dependsOn; + } else if (typeof dependsOn === 'string' && dependsOn.length > 0) { + parsed = dependsOn.split(',').map(s => s.trim()).filter(Boolean); + } else { + parsed = []; + } + + const result = await dependencyManager.validateDependencies(serviceId, parsed); + success(res, result); + }, 'dep-validate')); + + // ------------------------------------------------------------------------- + // GET /dependencies/:serviceId — Direct deps for one service + // ------------------------------------------------------------------------- + router.get('/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + const dependencies = await dependencyManager.getDependencies(serviceId); + const dependents = await dependencyManager.getDependents(serviceId); + + // Read the service's current dependsOn array + const services = await servicesStateManager.read(); + const allServices = Array.isArray(services) ? services : (services.services || []); + const service = allServices.find(s => s.id === serviceId); + + if (!service) { + throw new NotFoundError(`Service "${serviceId}"`); + } + + success(res, { + serviceId, + dependsOn: service.dependsOn || [], + dependencies, + dependents: dependents.map(d => ({ id: d.id, name: d.name })), + }); + }, 'dep-get')); + + // ------------------------------------------------------------------------- + // GET /dependencies/:serviceId/chain — Ordered restart chain + // ------------------------------------------------------------------------- + router.get('/:serviceId/chain', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + const chain = await dependencyManager.getOrderedRestartChain(serviceId); + success(res, { serviceId, chain }); + }, 'dep-chain')); + + // ------------------------------------------------------------------------- + // GET /dependencies/:serviceId/status — Dependency health status + // ------------------------------------------------------------------------- + router.get('/:serviceId/status', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + const statuses = await dependencyManager.getDependencyStatus(serviceId); + success(res, { serviceId, statuses }); + }, 'dep-status')); + + // ------------------------------------------------------------------------- + // POST /dependencies/:serviceId — Set dependencies + // ------------------------------------------------------------------------- + router.post('/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + const { dependsOn } = req.body; + + if (!Array.isArray(dependsOn)) { + throw new ValidationError('Request body must include dependsOn as an array of service IDs'); + } + + // Validate first + const validation = await dependencyManager.validateDependencies(serviceId, dependsOn); + if (!validation.valid) { + return errorResponse(res, validation.errors.join('; '), 400); + } + + // Update the service + let found = false; + await servicesStateManager.update(services => { + const arr = Array.isArray(services) ? services : []; + return arr.map(s => { + if (s.id === serviceId) { + found = true; + return { ...s, dependsOn: dependsOn.slice() }; + } + return s; + }); + }); + + if (!found) { + throw new NotFoundError(`Service "${serviceId}"`); + } + + log.info('dependency', 'Dependencies updated', { serviceId, dependsOn }); + + success(res, { + message: `Dependencies updated for "${serviceId}"`, + serviceId, + dependsOn, + }); + }, 'dep-set')); + + // ------------------------------------------------------------------------- + // DELETE /dependencies/:serviceId — Remove all dependencies for a service + // ------------------------------------------------------------------------- + router.delete('/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + let found = false; + await servicesStateManager.update(services => { + const arr = Array.isArray(services) ? services : []; + return arr.map(s => { + if (s.id === serviceId) { + found = true; + const updated = { ...s }; + delete updated.dependsOn; + return updated; + } + return s; + }); + }); + + if (!found) { + throw new NotFoundError(`Service "${serviceId}"`); + } + + log.info('dependency', 'Dependencies removed', { serviceId }); + + success(res, { + message: `All dependencies removed for "${serviceId}"`, + serviceId, + }); + }, 'dep-delete')); + + // ------------------------------------------------------------------------- + // POST /dependencies/:serviceId/restart — Restart with dependency chain + // ------------------------------------------------------------------------- + router.post('/:serviceId/restart', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + // Verify the service exists + const services = await servicesStateManager.read(); + const allServices = Array.isArray(services) ? services : (services.services || []); + if (!allServices.find(s => s.id === serviceId)) { + throw new NotFoundError(`Service "${serviceId}"`); + } + + // Get the chain first for the response (before async restart begins) + let chain; + try { + chain = await dependencyManager.getOrderedRestartChain(serviceId); + } catch (err) { + return errorResponse(res, err.message, 400); + } + + // Respond immediately with the chain order + success(res, { + message: `Dependency restart initiated for "${serviceId}"`, + serviceId, + chain, + }); + + // Run the restart chain asynchronously so the client doesn't block + dependencyManager.restartWithDependencies(serviceId).catch(err => { + if (log) { + log.error('dependency', 'Async dependency restart failed', { + serviceId, + error: err.message, + }); + } + }); + }, 'dep-restart')); + + return router; +}; diff --git a/dashcaddy-api/routes/dns.js b/dashcaddy-api/routes/dns.js index 3228bf6..0cf4951 100644 --- a/dashcaddy-api/routes/dns.js +++ b/dashcaddy-api/routes/dns.js @@ -26,7 +26,8 @@ module.exports = function({ log, safeErrorMessage, fetchT, - credentialManager + credentialManager, + dnsPropagationChecker }) { const router = express.Router(); @@ -139,6 +140,14 @@ module.exports = function({ }); if (result.status === 'ok') { + // Start DNS propagation verification in background + if (dnsPropagationChecker && ip) { + const fullDomain = domain; + dnsPropagationChecker.startVerification(fullDomain, ip).catch(err => { + log('DNS propagation check start failed:', err.message); + }); + } + success(res, { message: `DNS record ${domain} -> ${ip} created` }); } else { // Error handled by middleware @@ -641,5 +650,68 @@ module.exports = function({ } }, 'dns-update')); + // ===== DNS PROPAGATION ===== + + // GET /propagation — Get all recent DNS propagation checks + router.get('/propagation', asyncHandler(async (req, res) => { + if (!dnsPropagationChecker) { + return success(res, { verifications: [], message: 'DNS propagation checker not available' }); + } + + // Cleanup old entries + dnsPropagationChecker.cleanup(); + + const verifications = dnsPropagationChecker.getAllVerifications(); + success(res, { verifications }); + }, 'dns-propagation-all')); + + // POST /propagation/verify — Manually trigger DNS propagation verification + router.post('/propagation/verify', asyncHandler(async (req, res) => { + if (!dnsPropagationChecker) { + return errorResponse(res, 'DNS propagation checker not available', 503); + } + + const { domain, expectedIp } = req.body; + + if (!domain || !expectedIp) { + throw new ValidationError('domain and expectedIp are required'); + } + + // Validate domain format + if (!REGEX.DOMAIN.test(domain)) { + throw new ValidationError('[DC-301] Invalid domain format'); + } + + // Validate IP address + const validatorLib = require('validator'); + if (!validatorLib.isIP(expectedIp)) { + throw new ValidationError('[DC-210] Invalid IP address'); + } + + const job = dnsPropagationChecker.startVerification(domain, expectedIp); + success(res, { + message: 'DNS propagation verification started', + domain, + expectedIp, + status: job.status + }); + }, 'dns-propagation-verify')); + + // GET /propagation/:domain — Get propagation status for a specific domain + router.get('/propagation/:domain', asyncHandler(async (req, res) => { + if (!dnsPropagationChecker) { + return success(res, { verification: null, message: 'DNS propagation checker not available' }); + } + + const { domain } = req.params; + const status = dnsPropagationChecker.getVerificationStatus(domain); + + if (!status) { + throw new NotFoundError(`No propagation check found for domain: ${domain}`); + } + + success(res, { verification: status }); + }, 'dns-propagation-domain')); + return router; }; diff --git a/dashcaddy-api/routes/events.js b/dashcaddy-api/routes/events.js index 4a45455..9827f4e 100644 --- a/dashcaddy-api/routes/events.js +++ b/dashcaddy-api/routes/events.js @@ -8,9 +8,10 @@ const express = require('express'); * @param {Object} deps.healthChecker - Health checker * @param {Object} deps.updateManager - Update manager * @param {Function} deps.logError - Error logging function + * @param {Object} deps.dependencyManager - Dependency manager for restart chain events * @returns {express.Router} */ -module.exports = function({ resourceMonitor, healthChecker, updateManager, logError }) { +module.exports = function({ resourceMonitor, healthChecker, updateManager, logError, dependencyManager, autoRestartManager, driftDetector, sslMonitor, dnsPropagationChecker }) { const router = express.Router(); const clients = new Set(); @@ -74,6 +75,48 @@ module.exports = function({ resourceMonitor, healthChecker, updateManager, logEr }); } + // Dependency manager events + if (dependencyManager) { + dependencyManager.on('dependency-restart-start', (data) => { + broadcast('dependency-restart-start', data); + }); + dependencyManager.on('dependency-restart-progress', (data) => { + broadcast('dependency-restart-progress', data); + }); + dependencyManager.on('dependency-restart-complete', (data) => { + broadcast('dependency-restart-complete', data); + }); + dependencyManager.on('dependency-restart-failed', (data) => { + broadcast('dependency-restart-failed', data); + }); + } + + // Auto-restart manager events + if (autoRestartManager) { + autoRestartManager.on('auto-restart-attempt', (data) => broadcast('auto-restart-attempt', data)); + autoRestartManager.on('auto-restart-success', (data) => broadcast('auto-restart-success', data)); + autoRestartManager.on('auto-restart-failed', (data) => broadcast('auto-restart-failed', data)); + autoRestartManager.on('auto-restart-max-reached', (data) => broadcast('auto-restart-max-reached', data)); + } + + // Config drift detector events + if (driftDetector) { + driftDetector.on('drift-detected', (data) => broadcast('drift-detected', data)); + } + + // SSL monitor events + if (sslMonitor) { + sslMonitor.on('cert-expiring', (data) => broadcast('cert-expiring', data)); + sslMonitor.on('cert-critical', (data) => broadcast('cert-critical', data)); + } + + // DNS propagation checker events + if (dnsPropagationChecker) { + dnsPropagationChecker.on('propagation-check', (data) => broadcast('dns-propagation-check', data)); + dnsPropagationChecker.on('propagation-complete', (data) => broadcast('dns-propagation-complete', data)); + dnsPropagationChecker.on('propagation-timeout', (data) => broadcast('dns-propagation-timeout', data)); + } + // SSE endpoint router.get('/stream', (req, res) => { res.writeHead(200, { diff --git a/dashcaddy-api/routes/ssl-monitor.js b/dashcaddy-api/routes/ssl-monitor.js new file mode 100644 index 0000000..ffe53fa --- /dev/null +++ b/dashcaddy-api/routes/ssl-monitor.js @@ -0,0 +1,113 @@ +/** + * SSL Monitor Routes + * REST API endpoints for SSL certificate monitoring. + * + * @module routes/ssl-monitor + */ + +const express = require('express'); +const { success, error: errorResponse, notFound } = require('../response-helpers'); + +/** + * SSL Monitor route factory + * @param {Object} deps - Explicit dependencies + * @param {Object} deps.sslMonitor - SSLMonitor instance + * @param {Function} deps.asyncHandler - Async route handler wrapper + * @param {Function} deps.logError - Error logging function + * @returns {express.Router} + */ +module.exports = function({ sslMonitor, asyncHandler, logError }) { + const router = express.Router(); + + /** + * GET /ssl/certificates + * Get all SSL certificate statuses + */ + router.get('/certificates', asyncHandler(async (req, res) => { + const status = sslMonitor.getStatus(); + success(res, { certificates: status }); + }, 'ssl-certificates')); + + /** + * GET /ssl/certificates/:serviceId + * Get SSL certificate status for a specific service + */ + router.get('/certificates/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + const certStatus = sslMonitor.getServiceCertStatus(serviceId); + + if (!certStatus) { + return notFound(res, `No SSL certificate status found for service: ${serviceId}`); + } + + success(res, { certificate: certStatus }); + }, 'ssl-certificate-service')); + + /** + * POST /ssl/check + * Trigger an on-demand check of all SSL certificates + */ + router.post('/check', asyncHandler(async (req, res) => { + const results = await sslMonitor.checkAll(); + success(res, { certificates: results, message: 'SSL check completed' }); + }, 'ssl-check-all')); + + /** + * POST /ssl/check/:serviceId + * Check the SSL certificate for a specific service + */ + router.post('/check/:serviceId', asyncHandler(async (req, res) => { + const { serviceId } = req.params; + + // Look up the existing cert status to find the hostname + const existingCert = sslMonitor.getServiceCertStatus(serviceId); + if (!existingCert) { + return notFound(res, `No HTTPS URL found for service: ${serviceId}`); + } + + try { + const result = await sslMonitor.checkCert(existingCert.hostname, existingCert.port); + success(res, { certificate: { ...result, serviceId } }); + } catch (err) { + errorResponse(res, `Failed to check SSL certificate: ${err.message}`, 500); + } + }, 'ssl-check-service')); + + /** + * GET /ssl/config + * Get current SSL monitoring configuration + */ + router.get('/config', asyncHandler(async (req, res) => { + const config = sslMonitor.getConfig(); + success(res, { config }); + }, 'ssl-config-get')); + + /** + * POST /ssl/config + * Update SSL monitoring configuration + * Body: { enabled: boolean, intervalMs: number } + */ + router.post('/config', asyncHandler(async (req, res) => { + const { enabled, intervalMs } = req.body; + + // Validate inputs + if (enabled !== undefined && typeof enabled !== 'boolean') { + return errorResponse(res, 'enabled must be a boolean', 400); + } + if (intervalMs !== undefined) { + if (typeof intervalMs !== 'number' || intervalMs < 60000) { + return errorResponse(res, 'intervalMs must be a number >= 60000 (1 minute)', 400); + } + } + + const updates = {}; + if (enabled !== undefined) updates.enabled = enabled; + if (intervalMs !== undefined) updates.intervalMs = intervalMs; + + sslMonitor.updateConfig(updates); + const config = sslMonitor.getConfig(); + success(res, { config, message: 'SSL monitoring config updated' }); + }, 'ssl-config-update')); + + return router; +}; diff --git a/dashcaddy-api/src/app.js b/dashcaddy-api/src/app.js index 82ecf7e..bd60d68 100644 --- a/dashcaddy-api/src/app.js +++ b/dashcaddy-api/src/app.js @@ -77,6 +77,15 @@ const themesRoutes = require('../routes/themes'); const dockerResourcesRoutes = require('../routes/docker-resources'); const eventsRoutes = require('../routes/events'); const workflowsRoutes = require('../routes/workflows'); +const dependenciesRoutes = require('../routes/dependencies'); +const DependencyManager = require('../dependency-manager'); +const autoRestartRoutes = require('../routes/auto-restart'); +const configDriftRoutes = require('../routes/config-drift'); +const sslMonitorRoutes = require('../routes/ssl-monitor'); +const { AutoRestartManager } = require('../auto-restart-manager'); +const { ConfigDriftDetector } = require('../config-drift-detector'); +const { SSLMonitor } = require('../ssl-monitor'); +const { DNSPropagationChecker } = require('../dns-propagation'); // Constants const { APP } = require('../constants'); @@ -335,6 +344,39 @@ async function createApp() { } } + // Initialize dependency manager + const dependencyManager = new DependencyManager({ + servicesStateManager, + docker: ctx.docker, + notification: ctx.notification, + log, + }); + ctx.dependencyManager = dependencyManager; + log.info('app', 'Dependency manager initialized'); + + // Initialize auto-restart manager + const autoRestartManager = new AutoRestartManager(ctx); + ctx.autoRestartManager = autoRestartManager; + autoRestartManager.start(); + log.info('app', 'Auto-restart manager initialized'); + + // Initialize config drift detector + const driftDetector = new ConfigDriftDetector(ctx); + ctx.driftDetector = driftDetector; + driftDetector.startPolling(300000); // 5 min + log.info('app', 'Config drift detector initialized'); + + // Initialize SSL monitor + const sslMonitor = new SSLMonitor(ctx); + ctx.sslMonitor = sslMonitor; + sslMonitor.start(3600000); // 1 hour + log.info('app', 'SSL monitor initialized'); + + // Initialize DNS propagation checker + const dnsPropagationChecker = new DNSPropagationChecker(ctx); + ctx.dnsPropagationChecker = dnsPropagationChecker; + log.info('app', 'DNS propagation checker initialized'); + // Build versioned API router const apiRouter = express.Router(); @@ -375,7 +417,8 @@ async function createApp() { log: ctx.log, safeErrorMessage: ctx.safeErrorMessage, fetchT: ctx.fetchT, - credentialManager: ctx.credentialManager + credentialManager: ctx.credentialManager, + dnsPropagationChecker: ctx.dnsPropagationChecker })); apiRouter.use('/notifications', notificationRoutes({ notification: ctx.notification, @@ -489,13 +532,42 @@ async function createApp() { resourceMonitor: ctx.resourceMonitor, healthChecker: ctx.healthChecker, updateManager: ctx.updateManager, - logError: ctx.logError + logError: ctx.logError, + dependencyManager: ctx.dependencyManager, + autoRestartManager: ctx.autoRestartManager, + driftDetector: ctx.driftDetector, + sslMonitor: ctx.sslMonitor, + dnsPropagationChecker: ctx.dnsPropagationChecker })); apiRouter.use(workflowsRoutes({ workflowEngine: ctx.workflowEngine, licenseManager: ctx.licenseManager, asyncHandler: ctx.asyncHandler })); + apiRouter.use('/dependencies', dependenciesRoutes({ + dependencyManager: ctx.dependencyManager, + servicesStateManager: ctx.servicesStateManager, + docker: ctx.docker, + asyncHandler: ctx.asyncHandler, + logError: ctx.logError, + resyncHealthChecker: ctx.resyncHealthChecker, + log: ctx.log, + })); + apiRouter.use(autoRestartRoutes({ + autoRestartManager: ctx.autoRestartManager, + asyncHandler: ctx.asyncHandler, + logError: ctx.logError, + })); + apiRouter.use(configDriftRoutes({ + driftDetector: ctx.driftDetector, + asyncHandler: ctx.asyncHandler, + logError: ctx.logError, + })); + apiRouter.use(sslMonitorRoutes({ + sslMonitor: ctx.sslMonitor, + asyncHandler: ctx.asyncHandler, + logError: ctx.logError, + })); // Inline API routes apiRouter.get('/health', (req, res) => { diff --git a/dashcaddy-api/ssl-monitor.js b/dashcaddy-api/ssl-monitor.js new file mode 100644 index 0000000..78eef6d --- /dev/null +++ b/dashcaddy-api/ssl-monitor.js @@ -0,0 +1,411 @@ +/** + * SSL Certificate Monitor + * Periodically checks SSL certificates on services with HTTPS URLs. + * Alerts at 30, 14, and 7 days before expiry. + * + * @module ssl-monitor + */ + +const tls = require('tls'); +const EventEmitter = require('events'); +const path = require('path'); +const { readJsonFile, writeJsonFile } = require('./fs-helpers'); +const { resolveServiceUrl } = require('./url-resolver'); + +/** Default check interval: 1 hour */ +const DEFAULT_INTERVAL_MS = 3600000; + +/** Alert thresholds in days */ +const THRESHOLDS = { + WARNING: 30, + URGENT: 14, + CRITICAL: 7 +}; + +/** TLS connection timeout in milliseconds */ +const TLS_TIMEOUT_MS = 10000; + +class SSLMonitor extends EventEmitter { + /** + * Create an SSLMonitor instance. + * @param {Object} ctx - Shared application context + * @param {Object} ctx.servicesStateManager - State manager for reading services + * @param {Function} ctx.buildServiceUrl - URL builder helper + * @param {Object} ctx.siteConfig - Site configuration + * @param {Object} ctx.notification - NotificationManager instance + * @param {Object} ctx.log - Logger instance + * @param {string} [ctx.SSL_CACHE_FILE] - Path to persist SSL cache + */ + constructor(ctx) { + super(); + this.ctx = ctx; + this.log = ctx.log || console; + + /** @type {Map} hostname → last cert check result */ + this.certStatus = new Map(); + + /** @type {Map} hostname → last notified threshold level */ + this.notifiedThresholds = new Map(); + + /** @type {Map} hostname → service ID mapping */ + this.hostnameToServiceId = new Map(); + + /** @type {NodeJS.Timeout|null} */ + this.intervalHandle = null; + + /** Current config */ + this.config = { + enabled: true, + intervalMs: DEFAULT_INTERVAL_MS + }; + + /** Cache file path */ + this.cacheFile = ctx.SSL_CACHE_FILE || + path.join(path.dirname(ctx.SERVICES_FILE || './data'), 'ssl-cache.json'); + } + + /** + * Check the SSL certificate for a given hostname and port. + * Connects via TLS with rejectUnauthorized: false to retrieve certificate info. + * + * @param {string} hostname - The hostname to check + * @param {number} [port=443] - The port to connect to + * @returns {Promise} Certificate information + */ + async checkCert(hostname, port = 443) { + return new Promise((resolve, reject) => { + const socket = tls.connect({ + host: hostname, + port, + rejectUnauthorized: false, + servername: hostname, + timeout: TLS_TIMEOUT_MS + }, () => { + try { + const cert = socket.getPeerCertificate(); + + if (!cert || Object.keys(cert).length === 0) { + socket.destroy(); + return reject(new Error(`No certificate returned for ${hostname}:${port}`)); + } + + const validFrom = new Date(cert.valid_from); + const validTo = new Date(cert.valid_to); + const now = new Date(); + const msRemaining = validTo.getTime() - now.getTime(); + const daysRemaining = Math.ceil(msRemaining / (1000 * 60 * 60 * 24)); + + const result = { + hostname, + port, + subject: cert.subject?.CN || cert.subject?.O || 'Unknown', + issuer: cert.issuer?.CN || cert.issuer?.O || 'Unknown', + validFrom: cert.valid_from, + validTo: cert.valid_to, + daysRemaining, + fingerprint: cert.fingerprint || null, + isExpiring: daysRemaining <= THRESHOLDS.WARNING, + checkedAt: new Date().toISOString() + }; + + socket.destroy(); + resolve(result); + } catch (err) { + socket.destroy(); + reject(err); + } + }); + + socket.on('error', (err) => { + reject(new Error(`TLS connect error for ${hostname}:${port}: ${err.message}`)); + }); + + socket.setTimeout(TLS_TIMEOUT_MS, () => { + socket.destroy(new Error(`TLS connection timeout for ${hostname}:${port}`)); + reject(new Error(`TLS connection timeout for ${hostname}:${port}`)); + }); + }); + } + + /** + * Check SSL certificates for all services that have HTTPS URLs. + * Reads services from ctx.servicesStateManager, resolves URLs, and checks each HTTPS cert. + * + * @returns {Promise} Map of hostname → cert status + */ + async checkAll() { + if (!this.config.enabled) { + this.log.info('ssl-monitor', 'SSL monitoring is disabled, skipping check'); + return this.getStatus(); + } + + let servicesData; + try { + servicesData = await this.ctx.servicesStateManager.read(); + } catch (err) { + this.log.error('ssl-monitor', 'Failed to read services', { error: err.message }); + return this.getStatus(); + } + + const services = Array.isArray(servicesData) ? servicesData : (servicesData.services || []); + + for (const service of services) { + const serviceId = service.id || service.name?.toLowerCase(); + if (!serviceId) continue; + + try { + const url = resolveServiceUrl(serviceId, service, this.ctx.siteConfig, this.ctx.buildServiceUrl); + if (!url) continue; + + const parsed = new URL(url); + if (parsed.protocol !== 'https:') continue; + + const hostname = parsed.hostname; + const port = parseInt(parsed.port) || 443; + + // Map hostname back to service ID + this.hostnameToServiceId.set(hostname, serviceId); + + const result = await this.checkCert(hostname, port); + + // Store result + this.certStatus.set(hostname, result); + + // Emit check event + this.emit('cert-check', { serviceId, hostname, result }); + + // Check alert thresholds + await this._checkAndNotify(hostname, result, serviceId); + } catch (err) { + this.log.warn('ssl-monitor', `Failed to check cert for service ${serviceId}`, { + error: err.message + }); + } + } + + // Persist results + await this._saveCache(); + + return this.getStatus(); + } + + /** + * Start periodic SSL certificate checking. + * + * @param {number} [intervalMs=3600000] - Check interval in milliseconds + */ + start(intervalMs) { + if (intervalMs !== undefined) { + this.config.intervalMs = intervalMs; + } + if (this.intervalHandle) { + this.log.warn('ssl-monitor', 'SSL monitor is already running'); + return; + } + + this.config.enabled = true; + + // Load cached data + this._loadCache().catch(err => { + this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message }); + }); + + // Initial check (non-blocking) + this.checkAll().catch(err => { + this.log.error('ssl-monitor', 'Initial SSL check failed', { error: err.message }); + }); + + // Schedule periodic checks + this.intervalHandle = setInterval(() => { + this.checkAll().catch(err => { + this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message }); + }); + }, this.config.intervalMs); + + this.log.info('ssl-monitor', 'SSL monitoring started', { + intervalMs: this.config.intervalMs + }); + } + + /** + * Stop periodic SSL certificate checking. + */ + stop() { + if (this.intervalHandle) { + clearInterval(this.intervalHandle); + this.intervalHandle = null; + } + this.config.enabled = false; + this.log.info('ssl-monitor', 'SSL monitoring stopped'); + } + + /** + * Get the current SSL certificate status for all checked hostnames. + * + * @returns {Object} Map of hostname → cert status + */ + getStatus() { + const status = {}; + for (const [hostname, cert] of this.certStatus.entries()) { + status[hostname] = { ...cert }; + } + return status; + } + + /** + * Get the SSL certificate status for a specific service. + * + * @param {string} serviceId - The service ID to look up + * @returns {Object|null} Certificate status or null if not found + */ + getServiceCertStatus(serviceId) { + // Find hostname mapped to this service + for (const [hostname, id] of this.hostnameToServiceId.entries()) { + if (id === serviceId) { + const cert = this.certStatus.get(hostname); + return cert ? { ...cert, serviceId } : null; + } + } + return null; + } + + /** + * Get current monitoring configuration. + * + * @returns {Object} Config with interval and enabled state + */ + getConfig() { + return { ...this.config }; + } + + /** + * Update monitoring configuration. + * + * @param {Object} updates - Config updates + * @param {boolean} [updates.enabled] - Enable/disable monitoring + * @param {number} [updates.intervalMs] - Check interval in milliseconds + */ + updateConfig(updates) { + if (typeof updates.enabled === 'boolean') { + this.config.enabled = updates.enabled; + if (!updates.enabled && this.intervalHandle) { + this.stop(); + } + } + if (typeof updates.intervalMs === 'number' && updates.intervalMs >= 60000) { + this.config.intervalMs = updates.intervalMs; + // Restart interval if running + if (this.intervalHandle) { + clearInterval(this.intervalHandle); + this.intervalHandle = setInterval(() => { + this.checkAll().catch(err => { + this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message }); + }); + }, this.config.intervalMs); + } + } + } + + // ===== Private Methods ===== + + /** + * Check alert thresholds and send notifications if thresholds are crossed. + * Only sends one notification per threshold per hostname. + * + * @param {string} hostname + * @param {Object} certResult + * @param {string} serviceId + */ + async _checkAndNotify(hostname, certResult, serviceId) { + const { daysRemaining } = certResult; + const key = hostname; + const lastNotified = this.notifiedThresholds.get(key) || Infinity; + + let level = null; + let eventType = null; + let message = null; + + if (daysRemaining <= THRESHOLDS.CRITICAL) { + level = THRESHOLDS.CRITICAL; + eventType = 'cert-critical'; + message = `🔒 CRITICAL: SSL certificate for ${hostname} expires in ${daysRemaining} days!`; + } else if (daysRemaining <= THRESHOLDS.URGENT) { + level = THRESHOLDS.URGENT; + eventType = 'cert-expiring'; + message = `⚠️ URGENT: SSL certificate for ${hostname} expires in ${daysRemaining} days`; + } else if (daysRemaining <= THRESHOLDS.WARNING) { + level = THRESHOLDS.WARNING; + eventType = 'cert-expiring'; + message = `⚠️ SSL certificate for ${hostname} expires in ${daysRemaining} days`; + } + + if (level !== null && level < lastNotified) { + // New threshold crossed — send notification + this.notifiedThresholds.set(key, level); + this.emit(eventType, { hostname, serviceId, daysRemaining, level }); + + if (this.ctx.notification) { + try { + await this.ctx.notification.send('ssl-cert-expiry', { + text: message, + hostname, + serviceId, + daysRemaining, + level, + validTo: certResult.validTo + }, level <= THRESHOLDS.CRITICAL ? 'error' : 'warning'); + } catch (err) { + this.log.error('ssl-monitor', 'Failed to send SSL notification', { error: err.message }); + } + } + } else if (level === null) { + // Cert is healthy — reset notification tracking + this.notifiedThresholds.delete(key); + } + } + + /** + * Persist cert status cache to disk. + */ + async _saveCache() { + try { + const data = { + lastChecked: new Date().toISOString(), + certs: {}, + hostnameToServiceId: Object.fromEntries(this.hostnameToServiceId) + }; + for (const [hostname, cert] of this.certStatus.entries()) { + data.certs[hostname] = cert; + } + await writeJsonFile(this.cacheFile, data); + } catch (err) { + this.log.warn('ssl-monitor', 'Failed to save SSL cache', { error: err.message }); + } + } + + /** + * Load cert status cache from disk. + */ + async _loadCache() { + try { + const data = await readJsonFile(this.cacheFile, null); + if (data && data.certs) { + for (const [hostname, cert] of Object.entries(data.certs)) { + this.certStatus.set(hostname, cert); + } + if (data.hostnameToServiceId) { + for (const [hostname, serviceId] of Object.entries(data.hostnameToServiceId)) { + this.hostnameToServiceId.set(hostname, serviceId); + } + } + this.log.info('ssl-monitor', 'Loaded SSL cache', { + certCount: this.certStatus.size + }); + } + } catch (err) { + this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message }); + } + } +} + +module.exports = SSLMonitor; diff --git a/scripts/backup-gitea-to-dropbox.sh b/scripts/backup-gitea-to-dropbox.sh new file mode 100644 index 0000000..788fc5d --- /dev/null +++ b/scripts/backup-gitea-to-dropbox.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# ============================================================================= +# DashCaddy Gitea — Off-host backup to Dropbox +# ============================================================================= +# - Stops gitea container briefly to ensure SQLite DB consistency +# - Syncs /var/lib/docker/volumes/gitea-data to dropbox:/Apps/dashcaddy-gitea-backups// +# - Date-stamped snapshots (one per day), kept for 7 days locally +# - Restarts gitea even if sync fails +# - Logs to /var/log/gitea-backup.log +# ============================================================================= +set -u # don't use -e: we want to always restart gitea + +LOG=/var/log/gitea-backup.log +DATA_SRC=/var/lib/docker/volumes/gitea-data/_data +DEST="dropbox:/Apps/dashcaddy-gitea-backups" +TODAY=$(date -u +%Y-%m-%d) +BACKUP_PATH="${DEST}/${TODAY}" +RETENTION_DAYS=7 + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG"; } + +log "=== Backup start ===" + +# 0. Sanity checks +if [ ! -d "$DATA_SRC" ]; then + log "ERROR: data dir $DATA_SRC missing" + exit 1 +fi + +# 1. Stop gitea to flush SQLite +log "Stopping gitea container..." +docker stop gitea >> "$LOG" 2>&1 +STOP_RC=$? +if [ $STOP_RC -ne 0 ]; then + log "WARNING: docker stop returned $STOP_RC — container may not be running" +fi + +# 2. Sync (use copy so source files are preserved as-is, no --delete) +log "Syncing $DATA_SRC -> $BACKUP_PATH" +rclone copy "$DATA_SRC" "$BACKUP_PATH" \ + --transfers 4 \ + --checkers 8 \ + --retries 3 \ + --low-level-retries 10 \ + --stats 30s \ + --log-file "$LOG" \ + --log-level INFO +SYNC_RC=$? + +# 3. Always restart gitea +log "Starting gitea container..." +docker start gitea >> "$LOG" 2>&1 +START_RC=$? + +# Wait for gitea to be ready +for i in {1..30}; do + if curl -sf http://localhost:3000/api/v1/version > /dev/null 2>&1; then + log "Gitea is up after ${i}s" + break + fi + sleep 1 +done + +# 4. Cleanup old backups (older than RETENTION_DAYS) +log "Pruning local + remote snapshots older than ${RETENTION_DAYS} days..." +CUTOFF=$(date -u -d "${RETENTION_DAYS} days ago" +%Y-%m-%d) +rclone lsf "$DEST/" --dirs-only 2>/dev/null | while read -r d; do + # rclone returns names with trailing / + name="${d%/}" + if [[ "$name" < "$CUTOFF" ]]; then + log " removing old: $name" + rclone purge "${DEST}/${name}" >> "$LOG" 2>&1 + fi +done + +# 5. Report +if [ $SYNC_RC -eq 0 ] && [ $START_RC -eq 0 ]; then + log "=== Backup OK ===" + exit 0 +else + log "=== Backup completed with errors (sync=$SYNC_RC, start=$START_RC) ===" + exit 1 +fi