feat: auto-restart policies, SSL monitoring, DNS propagation, dependency tracking, config drift detection
CI / Test & Lint (push) Has been cancelled
CI / Security audit (push) Has been cancelled

This commit is contained in:
Hermes
2026-06-10 14:43:46 -07:00
parent afcccf811e
commit 954be9e868
15 changed files with 3048 additions and 6 deletions
+164
View File
@@ -0,0 +1,164 @@
/**
* Auto-Restart Policy Routes
*
* CRUD endpoints for per-container auto-restart policies.
* Also provides a dry-run test endpoint.
*
* @module routes/auto-restart
*/
const express = require('express');
const { success } = require('../response-helpers');
const { ValidationError, NotFoundError } = require('../errors');
/**
* Auto-restart route factory
*
* @param {Object} deps - Explicit dependencies
* @param {Object} deps.autoRestartManager - AutoRestartManager instance
* @param {Function} deps.asyncHandler - Async route handler wrapper
* @param {Function} deps.logError - Error logging function
* @returns {express.Router}
*/
module.exports = function ({ autoRestartManager, asyncHandler, logError }) {
const router = express.Router();
/**
* GET /auto-restart/policies
* List all configured auto-restart policies.
*/
router.get('/policies', asyncHandler(async (_req, res) => {
const policies = autoRestartManager.listPolicies();
success(res, { policies });
}, 'auto-restart-list'));
/**
* GET /auto-restart/policies/:serviceId
* Get the restart policy for a single service.
*/
router.get('/policies/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
throw new ValidationError('Invalid service ID format');
}
const policy = autoRestartManager.getPolicy(serviceId);
if (!policy) {
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
}
success(res, { policy });
}, 'auto-restart-get'));
/**
* POST /auto-restart/policies/:serviceId
* Create or update a restart policy.
*
* Body: { enabled, maxRetries, retryIntervalMs, windowMinutes }
*/
router.post('/policies/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
throw new ValidationError('Invalid service ID format');
}
const { enabled, maxRetries, retryIntervalMs, windowMinutes } = req.body;
// Validate inputs
if (enabled !== undefined && typeof enabled !== 'boolean') {
throw new ValidationError('enabled must be a boolean');
}
if (maxRetries !== undefined) {
if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 100) {
throw new ValidationError('maxRetries must be an integer between 0 and 100');
}
}
if (retryIntervalMs !== undefined) {
if (!Number.isInteger(retryIntervalMs) || retryIntervalMs < 0 || retryIntervalMs > 3600000) {
throw new ValidationError('retryIntervalMs must be an integer between 0 and 3600000');
}
}
if (windowMinutes !== undefined) {
if (!Number.isInteger(windowMinutes) || windowMinutes < 0 || windowMinutes > 1440) {
throw new ValidationError('windowMinutes must be an integer between 0 and 1440');
}
}
const policy = await autoRestartManager.setPolicy(serviceId, {
...(enabled !== undefined && { enabled }),
...(maxRetries !== undefined && { maxRetries }),
...(retryIntervalMs !== undefined && { retryIntervalMs }),
...(windowMinutes !== undefined && { windowMinutes }),
});
success(res, { policy, message: `Policy ${serviceId} saved` });
}, 'auto-restart-set'));
/**
* DELETE /auto-restart/policies/:serviceId
* Remove a restart policy.
*/
router.delete('/policies/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
throw new ValidationError('Invalid service ID format');
}
const removed = await autoRestartManager.removePolicy(serviceId);
if (!removed) {
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
}
success(res, { message: `Policy for "${serviceId}" removed` });
}, 'auto-restart-delete'));
/**
* POST /auto-restart/policies/:serviceId/test
* Dry-run: simulate a restart attempt without actually restarting.
* Returns what *would* happen given the current policy state.
*/
router.post('/policies/:serviceId/test', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
throw new ValidationError('Invalid service ID format');
}
const policy = autoRestartManager.getPolicy(serviceId);
if (!policy) {
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
}
const now = Date.now();
const inCooldown = policy.cooldownUntil && now < policy.cooldownUntil;
const wouldRetry = !inCooldown && policy.currentRetries < policy.maxRetries;
const nextAttempt = policy.currentRetries + 1;
success(res, {
dryRun: true,
serviceId,
policy: {
enabled: policy.enabled,
currentRetries: policy.currentRetries,
maxRetries: policy.maxRetries,
cooldownUntil: policy.cooldownUntil,
inCooldown,
},
wouldRestart: policy.enabled && wouldRetry,
wouldMaxOut: !wouldRetry && !inCooldown,
nextAttempt: wouldRetry ? nextAttempt : null,
message: !policy.enabled
? 'Policy is disabled — no restart would occur'
: inCooldown
? `In cooldown until ${new Date(policy.cooldownUntil).toISOString()} — would skip`
: wouldRetry
? `Would attempt restart ${nextAttempt}/${policy.maxRetries}`
: `Max retries (${policy.maxRetries}) already reached — would enter cooldown`,
});
}, 'auto-restart-test'));
return router;
};
+92
View File
@@ -0,0 +1,92 @@
/**
* Config Drift Detection Routes
*
* API endpoints for running drift detection, reading cached reports,
* auto-fixing drift, and controlling periodic polling.
*
* @module routes/config-drift
*/
const express = require('express');
const { success } = require('../response-helpers');
const { ValidationError, NotFoundError } = require('../errors');
/**
* Config-drift route factory
*
* @param {Object} deps - Explicit dependencies
* @param {Object} deps.driftDetector - ConfigDriftDetector instance
* @param {Function} deps.asyncHandler - Async route handler wrapper
* @param {Function} deps.logError - Error logging function
* @returns {express.Router}
*/
module.exports = function ({ driftDetector, asyncHandler, logError }) {
const router = express.Router();
/**
* GET /config-drift/report
* Run a fresh drift detection and return the full report.
*/
router.get('/report', asyncHandler(async (_req, res) => {
const report = await driftDetector.detect();
success(res, { report });
}, 'drift-report'));
/**
* GET /config-drift/last
* Return the last cached drift report (no re-detection).
*/
router.get('/last', asyncHandler(async (_req, res) => {
if (!driftDetector.lastReport) {
throw new NotFoundError('No cached drift report — run detection first');
}
success(res, { report: driftDetector.lastReport });
}, 'drift-last'));
/**
* POST /config-drift/fix
* Auto-fix detected drift: remove stale records, flag unknown containers.
*/
router.post('/fix', asyncHandler(async (_req, res) => {
const result = await driftDetector.autoFix();
success(res, {
message: 'Auto-fix applied',
staleRemoved: result.staleRemoved,
unknownFlagged: result.unknownFlagged,
});
}, 'drift-fix'));
/**
* POST /config-drift/polling
* Enable or disable periodic drift detection polling.
*
* Body: { enabled: boolean, intervalMs?: number }
*/
router.post('/polling', asyncHandler(async (req, res) => {
const { enabled, intervalMs } = req.body;
if (typeof enabled !== 'boolean') {
throw new ValidationError('enabled must be a boolean');
}
if (intervalMs !== undefined) {
if (!Number.isInteger(intervalMs) || intervalMs < 10000 || intervalMs > 86400000) {
throw new ValidationError('intervalMs must be an integer between 10000 and 86400000 (10s 24h)');
}
}
if (enabled) {
driftDetector.startPolling(intervalMs || 300000);
success(res, {
message: 'Drift polling enabled',
intervalMs: intervalMs || 300000,
});
} else {
driftDetector.stopPolling();
success(res, { message: 'Drift polling disabled' });
}
}, 'drift-polling'));
return router;
};
+235
View File
@@ -0,0 +1,235 @@
/**
* Dependencies Route — REST API for service dependency tracking
*
* Endpoints:
* GET /dependencies/graph Full dependency graph
* GET /dependencies/validate Validate a proposed dep chain
* GET /dependencies/:serviceId Direct deps for one service
* GET /dependencies/:serviceId/chain Ordered restart chain
* GET /dependencies/:serviceId/status Dependency health status
* POST /dependencies/:serviceId Set dependencies
* DELETE /dependencies/:serviceId Remove all dependencies
* POST /dependencies/:serviceId/restart Restart with dependency chain
*
* @module routes/dependencies
*/
const express = require('express');
const { success, error: errorResponse } = require('../response-helpers');
const { NotFoundError, ValidationError } = require('../errors');
/**
* Dependencies route factory
*
* @param {Object} deps - Explicit dependencies
* @param {Object} deps.dependencyManager - DependencyManager instance
* @param {Object} deps.servicesStateManager - State manager for services.json
* @param {Object} deps.docker - Docker client wrapper
* @param {Function} deps.asyncHandler - Async route handler wrapper
* @param {Function} deps.logError - Error logging function
* @param {Function} deps.resyncHealthChecker - Health checker resync function
* @param {Object} deps.log - Logger instance
* @returns {express.Router}
*/
module.exports = function({
dependencyManager,
servicesStateManager,
docker,
asyncHandler,
logError,
resyncHealthChecker,
log,
}) {
const router = express.Router();
// -------------------------------------------------------------------------
// GET /dependencies/graph — Full dependency graph
// -------------------------------------------------------------------------
router.get('/graph', asyncHandler(async (req, res) => {
const graph = await dependencyManager.getDependencyGraph();
success(res, { graph });
}, 'dep-graph'));
// -------------------------------------------------------------------------
// GET /dependencies/validate — Validate a proposed dep chain (query params)
// -------------------------------------------------------------------------
router.get('/validate', asyncHandler(async (req, res) => {
const { serviceId, dependsOn } = req.query;
if (!serviceId) {
throw new ValidationError('serviceId query parameter is required');
}
// dependsOn may be a comma-separated string or already an array
let parsed;
if (Array.isArray(dependsOn)) {
parsed = dependsOn;
} else if (typeof dependsOn === 'string' && dependsOn.length > 0) {
parsed = dependsOn.split(',').map(s => s.trim()).filter(Boolean);
} else {
parsed = [];
}
const result = await dependencyManager.validateDependencies(serviceId, parsed);
success(res, result);
}, 'dep-validate'));
// -------------------------------------------------------------------------
// GET /dependencies/:serviceId — Direct deps for one service
// -------------------------------------------------------------------------
router.get('/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
const dependencies = await dependencyManager.getDependencies(serviceId);
const dependents = await dependencyManager.getDependents(serviceId);
// Read the service's current dependsOn array
const services = await servicesStateManager.read();
const allServices = Array.isArray(services) ? services : (services.services || []);
const service = allServices.find(s => s.id === serviceId);
if (!service) {
throw new NotFoundError(`Service "${serviceId}"`);
}
success(res, {
serviceId,
dependsOn: service.dependsOn || [],
dependencies,
dependents: dependents.map(d => ({ id: d.id, name: d.name })),
});
}, 'dep-get'));
// -------------------------------------------------------------------------
// GET /dependencies/:serviceId/chain — Ordered restart chain
// -------------------------------------------------------------------------
router.get('/:serviceId/chain', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
const chain = await dependencyManager.getOrderedRestartChain(serviceId);
success(res, { serviceId, chain });
}, 'dep-chain'));
// -------------------------------------------------------------------------
// GET /dependencies/:serviceId/status — Dependency health status
// -------------------------------------------------------------------------
router.get('/:serviceId/status', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
const statuses = await dependencyManager.getDependencyStatus(serviceId);
success(res, { serviceId, statuses });
}, 'dep-status'));
// -------------------------------------------------------------------------
// POST /dependencies/:serviceId — Set dependencies
// -------------------------------------------------------------------------
router.post('/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
const { dependsOn } = req.body;
if (!Array.isArray(dependsOn)) {
throw new ValidationError('Request body must include dependsOn as an array of service IDs');
}
// Validate first
const validation = await dependencyManager.validateDependencies(serviceId, dependsOn);
if (!validation.valid) {
return errorResponse(res, validation.errors.join('; '), 400);
}
// Update the service
let found = false;
await servicesStateManager.update(services => {
const arr = Array.isArray(services) ? services : [];
return arr.map(s => {
if (s.id === serviceId) {
found = true;
return { ...s, dependsOn: dependsOn.slice() };
}
return s;
});
});
if (!found) {
throw new NotFoundError(`Service "${serviceId}"`);
}
log.info('dependency', 'Dependencies updated', { serviceId, dependsOn });
success(res, {
message: `Dependencies updated for "${serviceId}"`,
serviceId,
dependsOn,
});
}, 'dep-set'));
// -------------------------------------------------------------------------
// DELETE /dependencies/:serviceId — Remove all dependencies for a service
// -------------------------------------------------------------------------
router.delete('/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
let found = false;
await servicesStateManager.update(services => {
const arr = Array.isArray(services) ? services : [];
return arr.map(s => {
if (s.id === serviceId) {
found = true;
const updated = { ...s };
delete updated.dependsOn;
return updated;
}
return s;
});
});
if (!found) {
throw new NotFoundError(`Service "${serviceId}"`);
}
log.info('dependency', 'Dependencies removed', { serviceId });
success(res, {
message: `All dependencies removed for "${serviceId}"`,
serviceId,
});
}, 'dep-delete'));
// -------------------------------------------------------------------------
// POST /dependencies/:serviceId/restart — Restart with dependency chain
// -------------------------------------------------------------------------
router.post('/:serviceId/restart', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
// Verify the service exists
const services = await servicesStateManager.read();
const allServices = Array.isArray(services) ? services : (services.services || []);
if (!allServices.find(s => s.id === serviceId)) {
throw new NotFoundError(`Service "${serviceId}"`);
}
// Get the chain first for the response (before async restart begins)
let chain;
try {
chain = await dependencyManager.getOrderedRestartChain(serviceId);
} catch (err) {
return errorResponse(res, err.message, 400);
}
// Respond immediately with the chain order
success(res, {
message: `Dependency restart initiated for "${serviceId}"`,
serviceId,
chain,
});
// Run the restart chain asynchronously so the client doesn't block
dependencyManager.restartWithDependencies(serviceId).catch(err => {
if (log) {
log.error('dependency', 'Async dependency restart failed', {
serviceId,
error: err.message,
});
}
});
}, 'dep-restart'));
return router;
};
+73 -1
View File
@@ -26,7 +26,8 @@ module.exports = function({
log,
safeErrorMessage,
fetchT,
credentialManager
credentialManager,
dnsPropagationChecker
}) {
const router = express.Router();
@@ -139,6 +140,14 @@ module.exports = function({
});
if (result.status === 'ok') {
// Start DNS propagation verification in background
if (dnsPropagationChecker && ip) {
const fullDomain = domain;
dnsPropagationChecker.startVerification(fullDomain, ip).catch(err => {
log('DNS propagation check start failed:', err.message);
});
}
success(res, { message: `DNS record ${domain} -> ${ip} created` });
} else {
// Error handled by middleware
@@ -641,5 +650,68 @@ module.exports = function({
}
}, 'dns-update'));
// ===== DNS PROPAGATION =====
// GET /propagation — Get all recent DNS propagation checks
router.get('/propagation', asyncHandler(async (req, res) => {
if (!dnsPropagationChecker) {
return success(res, { verifications: [], message: 'DNS propagation checker not available' });
}
// Cleanup old entries
dnsPropagationChecker.cleanup();
const verifications = dnsPropagationChecker.getAllVerifications();
success(res, { verifications });
}, 'dns-propagation-all'));
// POST /propagation/verify — Manually trigger DNS propagation verification
router.post('/propagation/verify', asyncHandler(async (req, res) => {
if (!dnsPropagationChecker) {
return errorResponse(res, 'DNS propagation checker not available', 503);
}
const { domain, expectedIp } = req.body;
if (!domain || !expectedIp) {
throw new ValidationError('domain and expectedIp are required');
}
// Validate domain format
if (!REGEX.DOMAIN.test(domain)) {
throw new ValidationError('[DC-301] Invalid domain format');
}
// Validate IP address
const validatorLib = require('validator');
if (!validatorLib.isIP(expectedIp)) {
throw new ValidationError('[DC-210] Invalid IP address');
}
const job = dnsPropagationChecker.startVerification(domain, expectedIp);
success(res, {
message: 'DNS propagation verification started',
domain,
expectedIp,
status: job.status
});
}, 'dns-propagation-verify'));
// GET /propagation/:domain — Get propagation status for a specific domain
router.get('/propagation/:domain', asyncHandler(async (req, res) => {
if (!dnsPropagationChecker) {
return success(res, { verification: null, message: 'DNS propagation checker not available' });
}
const { domain } = req.params;
const status = dnsPropagationChecker.getVerificationStatus(domain);
if (!status) {
throw new NotFoundError(`No propagation check found for domain: ${domain}`);
}
success(res, { verification: status });
}, 'dns-propagation-domain'));
return router;
};
+44 -1
View File
@@ -8,9 +8,10 @@ const express = require('express');
* @param {Object} deps.healthChecker - Health checker
* @param {Object} deps.updateManager - Update manager
* @param {Function} deps.logError - Error logging function
* @param {Object} deps.dependencyManager - Dependency manager for restart chain events
* @returns {express.Router}
*/
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError }) {
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError, dependencyManager, autoRestartManager, driftDetector, sslMonitor, dnsPropagationChecker }) {
const router = express.Router();
const clients = new Set();
@@ -74,6 +75,48 @@ module.exports = function({ resourceMonitor, healthChecker, updateManager, logEr
});
}
// Dependency manager events
if (dependencyManager) {
dependencyManager.on('dependency-restart-start', (data) => {
broadcast('dependency-restart-start', data);
});
dependencyManager.on('dependency-restart-progress', (data) => {
broadcast('dependency-restart-progress', data);
});
dependencyManager.on('dependency-restart-complete', (data) => {
broadcast('dependency-restart-complete', data);
});
dependencyManager.on('dependency-restart-failed', (data) => {
broadcast('dependency-restart-failed', data);
});
}
// Auto-restart manager events
if (autoRestartManager) {
autoRestartManager.on('auto-restart-attempt', (data) => broadcast('auto-restart-attempt', data));
autoRestartManager.on('auto-restart-success', (data) => broadcast('auto-restart-success', data));
autoRestartManager.on('auto-restart-failed', (data) => broadcast('auto-restart-failed', data));
autoRestartManager.on('auto-restart-max-reached', (data) => broadcast('auto-restart-max-reached', data));
}
// Config drift detector events
if (driftDetector) {
driftDetector.on('drift-detected', (data) => broadcast('drift-detected', data));
}
// SSL monitor events
if (sslMonitor) {
sslMonitor.on('cert-expiring', (data) => broadcast('cert-expiring', data));
sslMonitor.on('cert-critical', (data) => broadcast('cert-critical', data));
}
// DNS propagation checker events
if (dnsPropagationChecker) {
dnsPropagationChecker.on('propagation-check', (data) => broadcast('dns-propagation-check', data));
dnsPropagationChecker.on('propagation-complete', (data) => broadcast('dns-propagation-complete', data));
dnsPropagationChecker.on('propagation-timeout', (data) => broadcast('dns-propagation-timeout', data));
}
// SSE endpoint
router.get('/stream', (req, res) => {
res.writeHead(200, {
+113
View File
@@ -0,0 +1,113 @@
/**
* SSL Monitor Routes
* REST API endpoints for SSL certificate monitoring.
*
* @module routes/ssl-monitor
*/
const express = require('express');
const { success, error: errorResponse, notFound } = require('../response-helpers');
/**
* SSL Monitor route factory
* @param {Object} deps - Explicit dependencies
* @param {Object} deps.sslMonitor - SSLMonitor instance
* @param {Function} deps.asyncHandler - Async route handler wrapper
* @param {Function} deps.logError - Error logging function
* @returns {express.Router}
*/
module.exports = function({ sslMonitor, asyncHandler, logError }) {
const router = express.Router();
/**
* GET /ssl/certificates
* Get all SSL certificate statuses
*/
router.get('/certificates', asyncHandler(async (req, res) => {
const status = sslMonitor.getStatus();
success(res, { certificates: status });
}, 'ssl-certificates'));
/**
* GET /ssl/certificates/:serviceId
* Get SSL certificate status for a specific service
*/
router.get('/certificates/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
const certStatus = sslMonitor.getServiceCertStatus(serviceId);
if (!certStatus) {
return notFound(res, `No SSL certificate status found for service: ${serviceId}`);
}
success(res, { certificate: certStatus });
}, 'ssl-certificate-service'));
/**
* POST /ssl/check
* Trigger an on-demand check of all SSL certificates
*/
router.post('/check', asyncHandler(async (req, res) => {
const results = await sslMonitor.checkAll();
success(res, { certificates: results, message: 'SSL check completed' });
}, 'ssl-check-all'));
/**
* POST /ssl/check/:serviceId
* Check the SSL certificate for a specific service
*/
router.post('/check/:serviceId', asyncHandler(async (req, res) => {
const { serviceId } = req.params;
// Look up the existing cert status to find the hostname
const existingCert = sslMonitor.getServiceCertStatus(serviceId);
if (!existingCert) {
return notFound(res, `No HTTPS URL found for service: ${serviceId}`);
}
try {
const result = await sslMonitor.checkCert(existingCert.hostname, existingCert.port);
success(res, { certificate: { ...result, serviceId } });
} catch (err) {
errorResponse(res, `Failed to check SSL certificate: ${err.message}`, 500);
}
}, 'ssl-check-service'));
/**
* GET /ssl/config
* Get current SSL monitoring configuration
*/
router.get('/config', asyncHandler(async (req, res) => {
const config = sslMonitor.getConfig();
success(res, { config });
}, 'ssl-config-get'));
/**
* POST /ssl/config
* Update SSL monitoring configuration
* Body: { enabled: boolean, intervalMs: number }
*/
router.post('/config', asyncHandler(async (req, res) => {
const { enabled, intervalMs } = req.body;
// Validate inputs
if (enabled !== undefined && typeof enabled !== 'boolean') {
return errorResponse(res, 'enabled must be a boolean', 400);
}
if (intervalMs !== undefined) {
if (typeof intervalMs !== 'number' || intervalMs < 60000) {
return errorResponse(res, 'intervalMs must be a number >= 60000 (1 minute)', 400);
}
}
const updates = {};
if (enabled !== undefined) updates.enabled = enabled;
if (intervalMs !== undefined) updates.intervalMs = intervalMs;
sslMonitor.updateConfig(updates);
const config = sslMonitor.getConfig();
success(res, { config, message: 'SSL monitoring config updated' });
}, 'ssl-config-update'));
return router;
};