504 lines
16 KiB
JavaScript
504 lines
16 KiB
JavaScript
/**
|
|
* Auto-Restart Manager - Per-container restart policies with retry tracking
|
|
*
|
|
* When a container goes down, attempts automatic restart up to N times
|
|
* (configurable per-service). Sends notifications on each attempt and
|
|
* when max retries are exceeded. Integrates with HealthChecker events.
|
|
*
|
|
* @module auto-restart-manager
|
|
*/
|
|
|
|
const EventEmitter = require('events');
|
|
const path = require('path');
|
|
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
|
|
|
/**
|
|
* Default policy values applied when a new policy is created.
|
|
* @readonly
|
|
*/
|
|
const DEFAULT_POLICY = {
|
|
enabled: true,
|
|
maxRetries: 3,
|
|
retryIntervalMs: 5000,
|
|
windowMinutes: 10,
|
|
currentRetries: 0,
|
|
lastRestartAt: null,
|
|
cooldownUntil: null,
|
|
};
|
|
|
|
/**
|
|
* Manages automatic container restart policies and execution.
|
|
*
|
|
* @extends EventEmitter
|
|
*
|
|
* @fires AutoRestartManager#auto-restart-attempt
|
|
* @fires AutoRestartManager#auto-restart-success
|
|
* @fires AutoRestartManager#auto-restart-failed
|
|
* @fires AutoRestartManager#auto-restart-max-reached
|
|
*/
|
|
class AutoRestartManager extends EventEmitter {
|
|
/**
|
|
* @param {Object} ctx - Shared application context
|
|
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
|
* @param {Object} ctx.healthChecker - HealthChecker singleton
|
|
* @param {Object} ctx.notification - NotificationManager instance
|
|
* @param {Object} ctx.log - Logger instance
|
|
* @param {Function} ctx.logError - Error logging function
|
|
* @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir)
|
|
*/
|
|
constructor(ctx) {
|
|
super();
|
|
this.ctx = ctx;
|
|
this.log = ctx.log || console;
|
|
this.logError = ctx.logError || ((_ctx, err) => console.error(err));
|
|
this.docker = ctx.docker;
|
|
this.healthChecker = ctx.healthChecker;
|
|
this.notification = ctx.notification;
|
|
|
|
/** @type {Map<string, Object>} serviceId -> policy */
|
|
this.policies = new Map();
|
|
|
|
/** Path to the JSON file that persists policies */
|
|
this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json');
|
|
|
|
/** Track previous health status per service for transition detection */
|
|
this._previousHealth = new Map();
|
|
|
|
/** Bound handlers so we can remove them on stop() */
|
|
this._onStatusCheck = this._handleStatusCheck.bind(this);
|
|
this._started = false;
|
|
}
|
|
|
|
// ─── Lifecycle ────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Load persisted policies, then wire into HealthChecker events.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async start() {
|
|
if (this._started) return;
|
|
|
|
// Load persisted policies from disk
|
|
try {
|
|
const data = await readJsonFile(this.policiesFile, {});
|
|
for (const [serviceId, policy] of Object.entries(data)) {
|
|
this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy });
|
|
}
|
|
this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size });
|
|
} catch (err) {
|
|
this.log.error('auto-restart', 'Failed to load policies', { error: err.message });
|
|
}
|
|
|
|
// Listen to health checker status transitions
|
|
if (this.healthChecker) {
|
|
this.healthChecker.on('status-check', this._onStatusCheck);
|
|
}
|
|
|
|
this._started = true;
|
|
this.log.info('auto-restart', 'Manager started');
|
|
}
|
|
|
|
/**
|
|
* Remove event listeners and stop processing health events.
|
|
*/
|
|
stop() {
|
|
if (!this._started) return;
|
|
|
|
if (this.healthChecker) {
|
|
this.healthChecker.removeListener('status-check', this._onStatusCheck);
|
|
}
|
|
|
|
this._started = false;
|
|
this.log.info('auto-restart', 'Manager stopped');
|
|
}
|
|
|
|
// ─── Policy CRUD ─────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Create or update a restart policy for a service.
|
|
*
|
|
* @param {string} serviceId - Unique service identifier
|
|
* @param {Object} policy - Partial policy fields to merge
|
|
* @param {boolean} [policy.enabled=true]
|
|
* @param {number} [policy.maxRetries=3]
|
|
* @param {number} [policy.retryIntervalMs=5000]
|
|
* @param {number} [policy.windowMinutes=10]
|
|
* @returns {Promise<Object>} The resulting policy
|
|
* @throws {Error} If serviceId is invalid
|
|
*/
|
|
async setPolicy(serviceId, policy) {
|
|
if (!serviceId || typeof serviceId !== 'string') {
|
|
throw new Error('serviceId is required');
|
|
}
|
|
|
|
const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId };
|
|
|
|
const merged = {
|
|
...existing,
|
|
...policy,
|
|
serviceId,
|
|
// Never allow caller to override runtime counters directly
|
|
currentRetries: existing.currentRetries || 0,
|
|
lastRestartAt: existing.lastRestartAt,
|
|
cooldownUntil: existing.cooldownUntil,
|
|
};
|
|
|
|
this.policies.set(serviceId, merged);
|
|
await this._savePolicies();
|
|
|
|
this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled });
|
|
return { ...merged };
|
|
}
|
|
|
|
/**
|
|
* Retrieve the policy for a service.
|
|
*
|
|
* @param {string} serviceId
|
|
* @returns {Object|null} Policy object or null if none exists
|
|
*/
|
|
getPolicy(serviceId) {
|
|
const policy = this.policies.get(serviceId);
|
|
return policy ? { ...policy } : null;
|
|
}
|
|
|
|
/**
|
|
* Return all policies as an array.
|
|
* @returns {Object[]}
|
|
*/
|
|
listPolicies() {
|
|
return Array.from(this.policies.values()).map(p => ({ ...p }));
|
|
}
|
|
|
|
/**
|
|
* Remove a service's restart policy.
|
|
*
|
|
* @param {string} serviceId
|
|
* @returns {Promise<boolean>} true if a policy was removed
|
|
*/
|
|
async removePolicy(serviceId) {
|
|
if (!this.policies.has(serviceId)) return false;
|
|
|
|
this.policies.delete(serviceId);
|
|
await this._savePolicies();
|
|
|
|
this.log.info('auto-restart', 'Policy removed', { serviceId });
|
|
return true;
|
|
}
|
|
|
|
// ─── Core Restart Logic ──────────────────────────────────────────────
|
|
|
|
/**
|
|
* Called when a container is detected as down.
|
|
*
|
|
* Checks policy, cooldown, and retry count, then either attempts a
|
|
* Docker restart or notifies that max retries were exceeded.
|
|
*
|
|
* @param {string} serviceId - Service identifier
|
|
* @param {string} containerId - Docker container ID to restart
|
|
* @returns {Promise<Object>} Result of the operation
|
|
*/
|
|
async handleContainerDown(serviceId, containerId) {
|
|
const policy = this.policies.get(serviceId);
|
|
if (!policy) {
|
|
return { action: 'ignored', reason: 'no-policy' };
|
|
}
|
|
|
|
if (!policy.enabled) {
|
|
return { action: 'ignored', reason: 'disabled' };
|
|
}
|
|
|
|
// Check cooldown window
|
|
const now = Date.now();
|
|
if (policy.cooldownUntil && now < policy.cooldownUntil) {
|
|
this.log.info('auto-restart', 'Skipping — cooldown active', {
|
|
serviceId,
|
|
cooldownUntil: new Date(policy.cooldownUntil).toISOString(),
|
|
});
|
|
return { action: 'skipped', reason: 'cooldown' };
|
|
}
|
|
|
|
// Max retries exceeded — notify and enter cooldown
|
|
if (policy.currentRetries >= policy.maxRetries) {
|
|
const cooldownMs = policy.windowMinutes * 60 * 1000;
|
|
policy.cooldownUntil = now + cooldownMs;
|
|
policy.currentRetries = 0; // Reset so next window can try again
|
|
await this._savePolicies();
|
|
|
|
const eventData = {
|
|
serviceId,
|
|
containerId,
|
|
maxRetries: policy.maxRetries,
|
|
cooldownUntil: policy.cooldownUntil,
|
|
timestamp: new Date().toISOString(),
|
|
};
|
|
|
|
/**
|
|
* @event AutoRestartManager#auto-restart-max-reached
|
|
* @type {Object}
|
|
*/
|
|
this.emit('auto-restart-max-reached', eventData);
|
|
|
|
// Send notification
|
|
try {
|
|
await this._notify('auto-restart', {
|
|
containerName: serviceId,
|
|
message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`,
|
|
...eventData,
|
|
});
|
|
} catch (notifErr) {
|
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
|
}
|
|
|
|
return { action: 'max-reached', ...eventData };
|
|
}
|
|
|
|
// Wait for the configured retry interval before attempting
|
|
if (policy.retryIntervalMs > 0 && policy.lastRestartAt) {
|
|
const elapsed = now - new Date(policy.lastRestartAt).getTime();
|
|
if (elapsed < policy.retryIntervalMs) {
|
|
const waitMs = policy.retryIntervalMs - elapsed;
|
|
this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs });
|
|
await new Promise(resolve => setTimeout(resolve, waitMs));
|
|
}
|
|
}
|
|
|
|
// Attempt restart
|
|
policy.currentRetries += 1;
|
|
const attemptNum = policy.currentRetries;
|
|
const maxRetries = policy.maxRetries;
|
|
|
|
/**
|
|
* @event AutoRestartManager#auto-restart-attempt
|
|
* @type {Object}
|
|
*/
|
|
this.emit('auto-restart-attempt', {
|
|
serviceId,
|
|
containerId,
|
|
attempt: attemptNum,
|
|
maxRetries,
|
|
timestamp: new Date().toISOString(),
|
|
});
|
|
|
|
try {
|
|
if (!this.docker?.client) {
|
|
throw new Error('Docker client not available');
|
|
}
|
|
|
|
const container = this.docker.client.getContainer(containerId);
|
|
await container.start();
|
|
|
|
policy.lastRestartAt = new Date().toISOString();
|
|
await this._savePolicies();
|
|
|
|
const successData = {
|
|
serviceId,
|
|
containerId,
|
|
attempt: attemptNum,
|
|
maxRetries,
|
|
timestamp: new Date().toISOString(),
|
|
};
|
|
|
|
/**
|
|
* @event AutoRestartManager#auto-restart-success
|
|
* @type {Object}
|
|
*/
|
|
this.emit('auto-restart-success', successData);
|
|
|
|
// Notify
|
|
try {
|
|
await this._notify('auto-restart', {
|
|
containerName: serviceId,
|
|
message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`,
|
|
...successData,
|
|
});
|
|
} catch (notifErr) {
|
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
|
}
|
|
|
|
this.log.info('auto-restart', 'Container restarted', {
|
|
serviceId,
|
|
attempt: attemptNum,
|
|
maxRetries,
|
|
});
|
|
|
|
return { action: 'restarted', ...successData };
|
|
} catch (restartErr) {
|
|
policy.lastRestartAt = new Date().toISOString();
|
|
await this._savePolicies();
|
|
|
|
const failData = {
|
|
serviceId,
|
|
containerId,
|
|
attempt: attemptNum,
|
|
maxRetries,
|
|
error: restartErr.message,
|
|
timestamp: new Date().toISOString(),
|
|
};
|
|
|
|
/**
|
|
* @event AutoRestartManager#auto-restart-failed
|
|
* @type {Object}
|
|
*/
|
|
this.emit('auto-restart-failed', failData);
|
|
|
|
// Notify
|
|
try {
|
|
await this._notify('auto-restart', {
|
|
containerName: serviceId,
|
|
message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`,
|
|
...failData,
|
|
});
|
|
} catch (notifErr) {
|
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
|
}
|
|
|
|
this.log.error('auto-restart', 'Restart failed', {
|
|
serviceId,
|
|
attempt: attemptNum,
|
|
error: restartErr.message,
|
|
});
|
|
|
|
return { action: 'failed', ...failData };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Called when a container recovers to healthy state.
|
|
* Resets the retry counter for the associated service.
|
|
*
|
|
* @param {string} serviceId
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async handleContainerUp(serviceId) {
|
|
const policy = this.policies.get(serviceId);
|
|
if (!policy) return;
|
|
|
|
if (policy.currentRetries > 0) {
|
|
policy.currentRetries = 0;
|
|
policy.cooldownUntil = null;
|
|
await this._savePolicies();
|
|
|
|
this.log.info('auto-restart', 'Retries reset after recovery', { serviceId });
|
|
}
|
|
}
|
|
|
|
// ─── Health Event Bridge ─────────────────────────────────────────────
|
|
|
|
/**
|
|
* Internal handler for HealthChecker `status-check` events.
|
|
* Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services.
|
|
*
|
|
* @param {Object} status - HealthChecker status object
|
|
* @param {string} status.serviceId
|
|
* @param {string} status.status - "up" or "down"
|
|
* @private
|
|
*/
|
|
async _handleStatusCheck(status) {
|
|
const { serviceId, status: currentStatus } = status;
|
|
if (!serviceId) return;
|
|
|
|
// Only process services that have a restart policy
|
|
if (!this.policies.has(serviceId)) return;
|
|
|
|
const previousStatus = this._previousHealth.get(serviceId);
|
|
this._previousHealth.set(serviceId, currentStatus);
|
|
|
|
// Transition: healthy → unhealthy
|
|
if (previousStatus === 'up' && currentStatus === 'down') {
|
|
// Find the containerId from the health checker config or status details
|
|
const containerId = this._resolveContainerId(serviceId, status);
|
|
if (containerId) {
|
|
try {
|
|
await this.handleContainerDown(serviceId, containerId);
|
|
} catch (err) {
|
|
this.logError('auto-restart-health-bridge', err);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Transition: unhealthy → healthy (recovery)
|
|
if (previousStatus === 'down' && currentStatus === 'up') {
|
|
try {
|
|
await this.handleContainerUp(serviceId);
|
|
} catch (err) {
|
|
this.logError('auto-restart-health-bridge', err);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempt to find the containerId for a service from various sources.
|
|
*
|
|
* @param {string} serviceId
|
|
* @param {Object} status - The status-check event data
|
|
* @returns {string|null}
|
|
* @private
|
|
*/
|
|
_resolveContainerId(serviceId, status) {
|
|
// Check if it's in the status details (some health checks embed it)
|
|
if (status.details?.containerId) return status.details.containerId;
|
|
|
|
// Look in the health checker config
|
|
const hcService = this.healthChecker?.config?.services?.[serviceId];
|
|
if (hcService?.containerId) return hcService.containerId;
|
|
|
|
// Try to look it up from the services state manager
|
|
try {
|
|
const servicesStateManager = this.ctx.servicesStateManager;
|
|
if (servicesStateManager) {
|
|
const readResult = servicesStateManager.read();
|
|
if (readResult && typeof readResult.then === 'function') {
|
|
// It returns a promise — fire-and-forget lookup
|
|
readResult.then(list => {
|
|
const found = (list || []).find(s => s.id === serviceId);
|
|
return found?.containerId || null;
|
|
}).catch(() => null);
|
|
} else {
|
|
const found = (readResult || []).find(s => s.id === serviceId);
|
|
if (found?.containerId) return found.containerId;
|
|
}
|
|
}
|
|
} catch (_) { /* best effort */ }
|
|
|
|
return null;
|
|
}
|
|
|
|
// ─── Persistence ─────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Persist current policies to disk.
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _savePolicies() {
|
|
try {
|
|
const obj = {};
|
|
for (const [serviceId, policy] of this.policies.entries()) {
|
|
obj[serviceId] = { ...policy };
|
|
}
|
|
await writeJsonFile(this.policiesFile, obj);
|
|
} catch (err) {
|
|
this.log.error('auto-restart', 'Failed to save policies', { error: err.message });
|
|
}
|
|
}
|
|
|
|
// ─── Helpers ─────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Send a notification via the notification manager.
|
|
*
|
|
* @param {string} event - Event type (e.g. 'auto-restart')
|
|
* @param {Object} data - Notification payload
|
|
* @returns {Promise<Object>}
|
|
* @private
|
|
*/
|
|
async _notify(event, data) {
|
|
if (this.notification?.send) {
|
|
return this.notification.send(event, data);
|
|
}
|
|
return { success: false, reason: 'no-notification-manager' };
|
|
}
|
|
}
|
|
|
|
module.exports = { AutoRestartManager, DEFAULT_POLICY };
|