feat: auto-restart policies, SSL monitoring, DNS propagation, dependency tracking, config drift detection
This commit is contained in:
Hermes
@@ -0,0 +1,503 @@
|
||||
/**
|
||||
* Auto-Restart Manager - Per-container restart policies with retry tracking
|
||||
*
|
||||
* When a container goes down, attempts automatic restart up to N times
|
||||
* (configurable per-service). Sends notifications on each attempt and
|
||||
* when max retries are exceeded. Integrates with HealthChecker events.
|
||||
*
|
||||
* @module auto-restart-manager
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const path = require('path');
|
||||
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
||||
|
||||
/**
|
||||
* Default policy values applied when a new policy is created.
|
||||
* @readonly
|
||||
*/
|
||||
const DEFAULT_POLICY = {
|
||||
enabled: true,
|
||||
maxRetries: 3,
|
||||
retryIntervalMs: 5000,
|
||||
windowMinutes: 10,
|
||||
currentRetries: 0,
|
||||
lastRestartAt: null,
|
||||
cooldownUntil: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Manages automatic container restart policies and execution.
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @fires AutoRestartManager#auto-restart-attempt
|
||||
* @fires AutoRestartManager#auto-restart-success
|
||||
* @fires AutoRestartManager#auto-restart-failed
|
||||
* @fires AutoRestartManager#auto-restart-max-reached
|
||||
*/
|
||||
class AutoRestartManager extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||
* @param {Object} ctx.healthChecker - HealthChecker singleton
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {Function} ctx.logError - Error logging function
|
||||
* @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir)
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
this.logError = ctx.logError || ((_ctx, err) => console.error(err));
|
||||
this.docker = ctx.docker;
|
||||
this.healthChecker = ctx.healthChecker;
|
||||
this.notification = ctx.notification;
|
||||
|
||||
/** @type {Map<string, Object>} serviceId -> policy */
|
||||
this.policies = new Map();
|
||||
|
||||
/** Path to the JSON file that persists policies */
|
||||
this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json');
|
||||
|
||||
/** Track previous health status per service for transition detection */
|
||||
this._previousHealth = new Map();
|
||||
|
||||
/** Bound handlers so we can remove them on stop() */
|
||||
this._onStatusCheck = this._handleStatusCheck.bind(this);
|
||||
this._started = false;
|
||||
}
|
||||
|
||||
// ─── Lifecycle ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Load persisted policies, then wire into HealthChecker events.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async start() {
|
||||
if (this._started) return;
|
||||
|
||||
// Load persisted policies from disk
|
||||
try {
|
||||
const data = await readJsonFile(this.policiesFile, {});
|
||||
for (const [serviceId, policy] of Object.entries(data)) {
|
||||
this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy });
|
||||
}
|
||||
this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size });
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to load policies', { error: err.message });
|
||||
}
|
||||
|
||||
// Listen to health checker status transitions
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.on('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = true;
|
||||
this.log.info('auto-restart', 'Manager started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listeners and stop processing health events.
|
||||
*/
|
||||
stop() {
|
||||
if (!this._started) return;
|
||||
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.removeListener('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = false;
|
||||
this.log.info('auto-restart', 'Manager stopped');
|
||||
}
|
||||
|
||||
// ─── Policy CRUD ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create or update a restart policy for a service.
|
||||
*
|
||||
* @param {string} serviceId - Unique service identifier
|
||||
* @param {Object} policy - Partial policy fields to merge
|
||||
* @param {boolean} [policy.enabled=true]
|
||||
* @param {number} [policy.maxRetries=3]
|
||||
* @param {number} [policy.retryIntervalMs=5000]
|
||||
* @param {number} [policy.windowMinutes=10]
|
||||
* @returns {Promise<Object>} The resulting policy
|
||||
* @throws {Error} If serviceId is invalid
|
||||
*/
|
||||
async setPolicy(serviceId, policy) {
|
||||
if (!serviceId || typeof serviceId !== 'string') {
|
||||
throw new Error('serviceId is required');
|
||||
}
|
||||
|
||||
const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId };
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...policy,
|
||||
serviceId,
|
||||
// Never allow caller to override runtime counters directly
|
||||
currentRetries: existing.currentRetries || 0,
|
||||
lastRestartAt: existing.lastRestartAt,
|
||||
cooldownUntil: existing.cooldownUntil,
|
||||
};
|
||||
|
||||
this.policies.set(serviceId, merged);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled });
|
||||
return { ...merged };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the policy for a service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Object|null} Policy object or null if none exists
|
||||
*/
|
||||
getPolicy(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
return policy ? { ...policy } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all policies as an array.
|
||||
* @returns {Object[]}
|
||||
*/
|
||||
listPolicies() {
|
||||
return Array.from(this.policies.values()).map(p => ({ ...p }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a service's restart policy.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<boolean>} true if a policy was removed
|
||||
*/
|
||||
async removePolicy(serviceId) {
|
||||
if (!this.policies.has(serviceId)) return false;
|
||||
|
||||
this.policies.delete(serviceId);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy removed', { serviceId });
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Core Restart Logic ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Called when a container is detected as down.
|
||||
*
|
||||
* Checks policy, cooldown, and retry count, then either attempts a
|
||||
* Docker restart or notifies that max retries were exceeded.
|
||||
*
|
||||
* @param {string} serviceId - Service identifier
|
||||
* @param {string} containerId - Docker container ID to restart
|
||||
* @returns {Promise<Object>} Result of the operation
|
||||
*/
|
||||
async handleContainerDown(serviceId, containerId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) {
|
||||
return { action: 'ignored', reason: 'no-policy' };
|
||||
}
|
||||
|
||||
if (!policy.enabled) {
|
||||
return { action: 'ignored', reason: 'disabled' };
|
||||
}
|
||||
|
||||
// Check cooldown window
|
||||
const now = Date.now();
|
||||
if (policy.cooldownUntil && now < policy.cooldownUntil) {
|
||||
this.log.info('auto-restart', 'Skipping — cooldown active', {
|
||||
serviceId,
|
||||
cooldownUntil: new Date(policy.cooldownUntil).toISOString(),
|
||||
});
|
||||
return { action: 'skipped', reason: 'cooldown' };
|
||||
}
|
||||
|
||||
// Max retries exceeded — notify and enter cooldown
|
||||
if (policy.currentRetries >= policy.maxRetries) {
|
||||
const cooldownMs = policy.windowMinutes * 60 * 1000;
|
||||
policy.cooldownUntil = now + cooldownMs;
|
||||
policy.currentRetries = 0; // Reset so next window can try again
|
||||
await this._savePolicies();
|
||||
|
||||
const eventData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
maxRetries: policy.maxRetries,
|
||||
cooldownUntil: policy.cooldownUntil,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-max-reached
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-max-reached', eventData);
|
||||
|
||||
// Send notification
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`,
|
||||
...eventData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
return { action: 'max-reached', ...eventData };
|
||||
}
|
||||
|
||||
// Wait for the configured retry interval before attempting
|
||||
if (policy.retryIntervalMs > 0 && policy.lastRestartAt) {
|
||||
const elapsed = now - new Date(policy.lastRestartAt).getTime();
|
||||
if (elapsed < policy.retryIntervalMs) {
|
||||
const waitMs = policy.retryIntervalMs - elapsed;
|
||||
this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs });
|
||||
await new Promise(resolve => setTimeout(resolve, waitMs));
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt restart
|
||||
policy.currentRetries += 1;
|
||||
const attemptNum = policy.currentRetries;
|
||||
const maxRetries = policy.maxRetries;
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-attempt
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-attempt', {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
if (!this.docker?.client) {
|
||||
throw new Error('Docker client not available');
|
||||
}
|
||||
|
||||
const container = this.docker.client.getContainer(containerId);
|
||||
await container.start();
|
||||
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const successData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-success
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-success', successData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`,
|
||||
...successData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.info('auto-restart', 'Container restarted', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
});
|
||||
|
||||
return { action: 'restarted', ...successData };
|
||||
} catch (restartErr) {
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const failData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
error: restartErr.message,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-failed
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-failed', failData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`,
|
||||
...failData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.error('auto-restart', 'Restart failed', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
error: restartErr.message,
|
||||
});
|
||||
|
||||
return { action: 'failed', ...failData };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a container recovers to healthy state.
|
||||
* Resets the retry counter for the associated service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async handleContainerUp(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) return;
|
||||
|
||||
if (policy.currentRetries > 0) {
|
||||
policy.currentRetries = 0;
|
||||
policy.cooldownUntil = null;
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Retries reset after recovery', { serviceId });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Health Event Bridge ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Internal handler for HealthChecker `status-check` events.
|
||||
* Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services.
|
||||
*
|
||||
* @param {Object} status - HealthChecker status object
|
||||
* @param {string} status.serviceId
|
||||
* @param {string} status.status - "up" or "down"
|
||||
* @private
|
||||
*/
|
||||
async _handleStatusCheck(status) {
|
||||
const { serviceId, status: currentStatus } = status;
|
||||
if (!serviceId) return;
|
||||
|
||||
// Only process services that have a restart policy
|
||||
if (!this.policies.has(serviceId)) return;
|
||||
|
||||
const previousStatus = this._previousHealth.get(serviceId);
|
||||
this._previousHealth.set(serviceId, currentStatus);
|
||||
|
||||
// Transition: healthy → unhealthy
|
||||
if (previousStatus === 'up' && currentStatus === 'down') {
|
||||
// Find the containerId from the health checker config or status details
|
||||
const containerId = this._resolveContainerId(serviceId, status);
|
||||
if (containerId) {
|
||||
try {
|
||||
await this.handleContainerDown(serviceId, containerId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transition: unhealthy → healthy (recovery)
|
||||
if (previousStatus === 'down' && currentStatus === 'up') {
|
||||
try {
|
||||
await this.handleContainerUp(serviceId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to find the containerId for a service from various sources.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @param {Object} status - The status-check event data
|
||||
* @returns {string|null}
|
||||
* @private
|
||||
*/
|
||||
_resolveContainerId(serviceId, status) {
|
||||
// Check if it's in the status details (some health checks embed it)
|
||||
if (status.details?.containerId) return status.details.containerId;
|
||||
|
||||
// Look in the health checker config
|
||||
const hcService = this.healthChecker?.config?.services?.[serviceId];
|
||||
if (hcService?.containerId) return hcService.containerId;
|
||||
|
||||
// Try to look it up from the services state manager
|
||||
try {
|
||||
const servicesStateManager = this.ctx.servicesStateManager;
|
||||
if (servicesStateManager) {
|
||||
const readResult = servicesStateManager.read();
|
||||
if (readResult && typeof readResult.then === 'function') {
|
||||
// It returns a promise — fire-and-forget lookup
|
||||
readResult.then(list => {
|
||||
const found = (list || []).find(s => s.id === serviceId);
|
||||
return found?.containerId || null;
|
||||
}).catch(() => null);
|
||||
} else {
|
||||
const found = (readResult || []).find(s => s.id === serviceId);
|
||||
if (found?.containerId) return found.containerId;
|
||||
}
|
||||
}
|
||||
} catch (_) { /* best effort */ }
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Persistence ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Persist current policies to disk.
|
||||
* @returns {Promise<void>}
|
||||
* @private
|
||||
*/
|
||||
async _savePolicies() {
|
||||
try {
|
||||
const obj = {};
|
||||
for (const [serviceId, policy] of this.policies.entries()) {
|
||||
obj[serviceId] = { ...policy };
|
||||
}
|
||||
await writeJsonFile(this.policiesFile, obj);
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to save policies', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Send a notification via the notification manager.
|
||||
*
|
||||
* @param {string} event - Event type (e.g. 'auto-restart')
|
||||
* @param {Object} data - Notification payload
|
||||
* @returns {Promise<Object>}
|
||||
* @private
|
||||
*/
|
||||
async _notify(event, data) {
|
||||
if (this.notification?.send) {
|
||||
return this.notification.send(event, data);
|
||||
}
|
||||
return { success: false, reason: 'no-notification-manager' };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { AutoRestartManager, DEFAULT_POLICY };
|
||||
Reference in New Issue
Block a user