/** * Auto-Restart Manager - Per-container restart policies with retry tracking * * When a container goes down, attempts automatic restart up to N times * (configurable per-service). Sends notifications on each attempt and * when max retries are exceeded. Integrates with HealthChecker events. * * @module auto-restart-manager */ const EventEmitter = require('events'); const path = require('path'); const { readJsonFile, writeJsonFile } = require('./fs-helpers'); /** * Default policy values applied when a new policy is created. * @readonly */ const DEFAULT_POLICY = { enabled: true, maxRetries: 3, retryIntervalMs: 5000, windowMinutes: 10, currentRetries: 0, lastRestartAt: null, cooldownUntil: null, }; /** * Manages automatic container restart policies and execution. * * @extends EventEmitter * * @fires AutoRestartManager#auto-restart-attempt * @fires AutoRestartManager#auto-restart-success * @fires AutoRestartManager#auto-restart-failed * @fires AutoRestartManager#auto-restart-max-reached */ class AutoRestartManager extends EventEmitter { /** * @param {Object} ctx - Shared application context * @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode }) * @param {Object} ctx.healthChecker - HealthChecker singleton * @param {Object} ctx.notification - NotificationManager instance * @param {Object} ctx.log - Logger instance * @param {Function} ctx.logError - Error logging function * @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir) */ constructor(ctx) { super(); this.ctx = ctx; this.log = ctx.log || console; this.logError = ctx.logError || ((_ctx, err) => console.error(err)); this.docker = ctx.docker; this.healthChecker = ctx.healthChecker; this.notification = ctx.notification; /** @type {Map} serviceId -> policy */ this.policies = new Map(); /** Path to the JSON file that persists policies */ this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json'); /** Track previous health status per service for transition detection */ this._previousHealth = new Map(); /** Bound handlers so we can remove them on stop() */ this._onStatusCheck = this._handleStatusCheck.bind(this); this._started = false; } // ─── Lifecycle ──────────────────────────────────────────────────────── /** * Load persisted policies, then wire into HealthChecker events. * @returns {Promise} */ async start() { if (this._started) return; // Load persisted policies from disk try { const data = await readJsonFile(this.policiesFile, {}); for (const [serviceId, policy] of Object.entries(data)) { this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy }); } this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size }); } catch (err) { this.log.error('auto-restart', 'Failed to load policies', { error: err.message }); } // Listen to health checker status transitions if (this.healthChecker) { this.healthChecker.on('status-check', this._onStatusCheck); } this._started = true; this.log.info('auto-restart', 'Manager started'); } /** * Remove event listeners and stop processing health events. */ stop() { if (!this._started) return; if (this.healthChecker) { this.healthChecker.removeListener('status-check', this._onStatusCheck); } this._started = false; this.log.info('auto-restart', 'Manager stopped'); } // ─── Policy CRUD ───────────────────────────────────────────────────── /** * Create or update a restart policy for a service. * * @param {string} serviceId - Unique service identifier * @param {Object} policy - Partial policy fields to merge * @param {boolean} [policy.enabled=true] * @param {number} [policy.maxRetries=3] * @param {number} [policy.retryIntervalMs=5000] * @param {number} [policy.windowMinutes=10] * @returns {Promise} The resulting policy * @throws {Error} If serviceId is invalid */ async setPolicy(serviceId, policy) { if (!serviceId || typeof serviceId !== 'string') { throw new Error('serviceId is required'); } const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId }; const merged = { ...existing, ...policy, serviceId, // Never allow caller to override runtime counters directly currentRetries: existing.currentRetries || 0, lastRestartAt: existing.lastRestartAt, cooldownUntil: existing.cooldownUntil, }; this.policies.set(serviceId, merged); await this._savePolicies(); this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled }); return { ...merged }; } /** * Retrieve the policy for a service. * * @param {string} serviceId * @returns {Object|null} Policy object or null if none exists */ getPolicy(serviceId) { const policy = this.policies.get(serviceId); return policy ? { ...policy } : null; } /** * Return all policies as an array. * @returns {Object[]} */ listPolicies() { return Array.from(this.policies.values()).map(p => ({ ...p })); } /** * Remove a service's restart policy. * * @param {string} serviceId * @returns {Promise} true if a policy was removed */ async removePolicy(serviceId) { if (!this.policies.has(serviceId)) return false; this.policies.delete(serviceId); await this._savePolicies(); this.log.info('auto-restart', 'Policy removed', { serviceId }); return true; } // ─── Core Restart Logic ────────────────────────────────────────────── /** * Called when a container is detected as down. * * Checks policy, cooldown, and retry count, then either attempts a * Docker restart or notifies that max retries were exceeded. * * @param {string} serviceId - Service identifier * @param {string} containerId - Docker container ID to restart * @returns {Promise} Result of the operation */ async handleContainerDown(serviceId, containerId) { const policy = this.policies.get(serviceId); if (!policy) { return { action: 'ignored', reason: 'no-policy' }; } if (!policy.enabled) { return { action: 'ignored', reason: 'disabled' }; } // Check cooldown window const now = Date.now(); if (policy.cooldownUntil && now < policy.cooldownUntil) { this.log.info('auto-restart', 'Skipping — cooldown active', { serviceId, cooldownUntil: new Date(policy.cooldownUntil).toISOString(), }); return { action: 'skipped', reason: 'cooldown' }; } // Max retries exceeded — notify and enter cooldown if (policy.currentRetries >= policy.maxRetries) { const cooldownMs = policy.windowMinutes * 60 * 1000; policy.cooldownUntil = now + cooldownMs; policy.currentRetries = 0; // Reset so next window can try again await this._savePolicies(); const eventData = { serviceId, containerId, maxRetries: policy.maxRetries, cooldownUntil: policy.cooldownUntil, timestamp: new Date().toISOString(), }; /** * @event AutoRestartManager#auto-restart-max-reached * @type {Object} */ this.emit('auto-restart-max-reached', eventData); // Send notification try { await this._notify('auto-restart', { containerName: serviceId, message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`, ...eventData, }); } catch (notifErr) { this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); } return { action: 'max-reached', ...eventData }; } // Wait for the configured retry interval before attempting if (policy.retryIntervalMs > 0 && policy.lastRestartAt) { const elapsed = now - new Date(policy.lastRestartAt).getTime(); if (elapsed < policy.retryIntervalMs) { const waitMs = policy.retryIntervalMs - elapsed; this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs }); await new Promise(resolve => setTimeout(resolve, waitMs)); } } // Attempt restart policy.currentRetries += 1; const attemptNum = policy.currentRetries; const maxRetries = policy.maxRetries; /** * @event AutoRestartManager#auto-restart-attempt * @type {Object} */ this.emit('auto-restart-attempt', { serviceId, containerId, attempt: attemptNum, maxRetries, timestamp: new Date().toISOString(), }); try { if (!this.docker?.client) { throw new Error('Docker client not available'); } const container = this.docker.client.getContainer(containerId); await container.start(); policy.lastRestartAt = new Date().toISOString(); await this._savePolicies(); const successData = { serviceId, containerId, attempt: attemptNum, maxRetries, timestamp: new Date().toISOString(), }; /** * @event AutoRestartManager#auto-restart-success * @type {Object} */ this.emit('auto-restart-success', successData); // Notify try { await this._notify('auto-restart', { containerName: serviceId, message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`, ...successData, }); } catch (notifErr) { this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); } this.log.info('auto-restart', 'Container restarted', { serviceId, attempt: attemptNum, maxRetries, }); return { action: 'restarted', ...successData }; } catch (restartErr) { policy.lastRestartAt = new Date().toISOString(); await this._savePolicies(); const failData = { serviceId, containerId, attempt: attemptNum, maxRetries, error: restartErr.message, timestamp: new Date().toISOString(), }; /** * @event AutoRestartManager#auto-restart-failed * @type {Object} */ this.emit('auto-restart-failed', failData); // Notify try { await this._notify('auto-restart', { containerName: serviceId, message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`, ...failData, }); } catch (notifErr) { this.log.error('auto-restart', 'Notification failed', { error: notifErr.message }); } this.log.error('auto-restart', 'Restart failed', { serviceId, attempt: attemptNum, error: restartErr.message, }); return { action: 'failed', ...failData }; } } /** * Called when a container recovers to healthy state. * Resets the retry counter for the associated service. * * @param {string} serviceId * @returns {Promise} */ async handleContainerUp(serviceId) { const policy = this.policies.get(serviceId); if (!policy) return; if (policy.currentRetries > 0) { policy.currentRetries = 0; policy.cooldownUntil = null; await this._savePolicies(); this.log.info('auto-restart', 'Retries reset after recovery', { serviceId }); } } // ─── Health Event Bridge ───────────────────────────────────────────── /** * Internal handler for HealthChecker `status-check` events. * Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services. * * @param {Object} status - HealthChecker status object * @param {string} status.serviceId * @param {string} status.status - "up" or "down" * @private */ async _handleStatusCheck(status) { const { serviceId, status: currentStatus } = status; if (!serviceId) return; // Only process services that have a restart policy if (!this.policies.has(serviceId)) return; const previousStatus = this._previousHealth.get(serviceId); this._previousHealth.set(serviceId, currentStatus); // Transition: healthy → unhealthy if (previousStatus === 'up' && currentStatus === 'down') { // Find the containerId from the health checker config or status details const containerId = this._resolveContainerId(serviceId, status); if (containerId) { try { await this.handleContainerDown(serviceId, containerId); } catch (err) { this.logError('auto-restart-health-bridge', err); } } } // Transition: unhealthy → healthy (recovery) if (previousStatus === 'down' && currentStatus === 'up') { try { await this.handleContainerUp(serviceId); } catch (err) { this.logError('auto-restart-health-bridge', err); } } } /** * Attempt to find the containerId for a service from various sources. * * @param {string} serviceId * @param {Object} status - The status-check event data * @returns {string|null} * @private */ _resolveContainerId(serviceId, status) { // Check if it's in the status details (some health checks embed it) if (status.details?.containerId) return status.details.containerId; // Look in the health checker config const hcService = this.healthChecker?.config?.services?.[serviceId]; if (hcService?.containerId) return hcService.containerId; // Try to look it up from the services state manager try { const servicesStateManager = this.ctx.servicesStateManager; if (servicesStateManager) { const readResult = servicesStateManager.read(); if (readResult && typeof readResult.then === 'function') { // It returns a promise — fire-and-forget lookup readResult.then(list => { const found = (list || []).find(s => s.id === serviceId); return found?.containerId || null; }).catch(() => null); } else { const found = (readResult || []).find(s => s.id === serviceId); if (found?.containerId) return found.containerId; } } } catch (_) { /* best effort */ } return null; } // ─── Persistence ───────────────────────────────────────────────────── /** * Persist current policies to disk. * @returns {Promise} * @private */ async _savePolicies() { try { const obj = {}; for (const [serviceId, policy] of this.policies.entries()) { obj[serviceId] = { ...policy }; } await writeJsonFile(this.policiesFile, obj); } catch (err) { this.log.error('auto-restart', 'Failed to save policies', { error: err.message }); } } // ─── Helpers ───────────────────────────────────────────────────────── /** * Send a notification via the notification manager. * * @param {string} event - Event type (e.g. 'auto-restart') * @param {Object} data - Notification payload * @returns {Promise} * @private */ async _notify(event, data) { if (this.notification?.send) { return this.notification.send(event, data); } return { success: false, reason: 'no-notification-manager' }; } } module.exports = { AutoRestartManager, DEFAULT_POLICY };