DC-005: Fix all 138 broken test paths after src/ refactor
After the DC-005 module reorganization (41 files moved into src/ subdirs),
138 test suites failed because the refactor script's path-rewrite logic
missed three categories:
1. Files inside src/ doing 'require("./src/...")' — should be 'require("../...")'
2. Files in src/X/Y/ doing 'require("../../../src/...")' — should be 'require("../../...")'
3. Test files in __tests__/ with leftover 'require("../../../src/...")' paths
Root cause: the original refactor script ran before all files were moved,
so it computed relative paths against stale filesystem state.
Result:
- 30/30 test suites pass
- 879/879 tests pass (was: 18/30 suites, 614/687 tests)
Also fixed:
- routes/apps/restore.js: wrong responses import path
- routes/*/*.js: '../../src/utilities/X' → '../src/utilities/X' (depth 2 routes)
This commit is contained in:
Hermes
@@ -0,0 +1,591 @@
|
||||
/**
|
||||
* Health Check Dashboard Module
|
||||
* Monitors service health, response times, and uptime
|
||||
* Provides SLA tracking and incident management
|
||||
*/
|
||||
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
const EventEmitter = require('events');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const HEALTH_CONFIG_FILE = process.env.HEALTH_CONFIG_FILE || path.join(__dirname, 'health-config.json');
|
||||
const HEALTH_HISTORY_FILE = process.env.HEALTH_HISTORY_FILE || path.join(__dirname, 'health-history.json');
|
||||
const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10); // 30 seconds
|
||||
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
||||
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
||||
|
||||
class HealthChecker extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.config = this.loadConfig();
|
||||
this.history = this.loadHistory();
|
||||
this.currentStatus = new Map();
|
||||
this.incidents = [];
|
||||
this.checking = false;
|
||||
this.checkInterval = null;
|
||||
this.consecutiveFailures = new Map(); // serviceId -> failure count
|
||||
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health checking
|
||||
*/
|
||||
start() {
|
||||
if (this.checking) return;
|
||||
|
||||
this.checking = true;
|
||||
|
||||
// Initial check
|
||||
this.checkAll();
|
||||
|
||||
// Schedule periodic checks
|
||||
this.checkInterval = setInterval(() => this.checkAll(), CHECK_INTERVAL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop health checking
|
||||
*/
|
||||
stop() {
|
||||
if (!this.checking) return;
|
||||
|
||||
this.checking = false;
|
||||
|
||||
if (this.checkInterval) {
|
||||
clearInterval(this.checkInterval);
|
||||
this.checkInterval = null;
|
||||
}
|
||||
|
||||
// Clear per-service backoff timers
|
||||
for (const timer of this.serviceTimers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.serviceTimers.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the backoff interval for a service based on consecutive failures.
|
||||
* Doubles the interval for each failure, capped at MAX_CHECK_INTERVAL.
|
||||
*/
|
||||
getBackoffInterval(serviceId) {
|
||||
const failures = this.consecutiveFailures.get(serviceId) || 0;
|
||||
if (failures === 0) return CHECK_INTERVAL;
|
||||
return Math.min(CHECK_INTERVAL * Math.pow(2, failures), MAX_CHECK_INTERVAL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check all configured services
|
||||
*/
|
||||
async checkAll() {
|
||||
const services = Object.entries(this.config.services || {});
|
||||
|
||||
for (const [serviceId, config] of services) {
|
||||
if (config.enabled !== false) {
|
||||
try {
|
||||
await this.checkService(serviceId, config);
|
||||
} catch (error) {
|
||||
// Error logged via checkForIncidents
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup old history
|
||||
this.cleanupHistory();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a single service
|
||||
*/
|
||||
async checkService(serviceId, config) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const result = await this.performHealthCheck(config);
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
const status = {
|
||||
serviceId,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: result.healthy ? 'up' : 'down',
|
||||
responseTime,
|
||||
statusCode: result.statusCode,
|
||||
message: result.message,
|
||||
details: result.details
|
||||
};
|
||||
|
||||
// Track consecutive failures for exponential backoff
|
||||
if (result.healthy) {
|
||||
this.consecutiveFailures.delete(serviceId);
|
||||
} else {
|
||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||
}
|
||||
|
||||
this.recordStatus(serviceId, status);
|
||||
this.checkForIncidents(serviceId, status, config);
|
||||
|
||||
return status;
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
// Increment failure count for backoff
|
||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||
|
||||
const status = {
|
||||
serviceId,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'down',
|
||||
responseTime,
|
||||
error: error.message
|
||||
};
|
||||
|
||||
this.recordStatus(serviceId, status);
|
||||
this.checkForIncidents(serviceId, status, config);
|
||||
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform actual health check
|
||||
*/
|
||||
async performHealthCheck(config) {
|
||||
const result = await this._doRequest(config, config.method || 'GET');
|
||||
// Fall back to GET if HEAD is not supported
|
||||
if ((result.statusCode === 501 || result.statusCode === 405) && (config.method || '').toUpperCase() === 'HEAD') {
|
||||
return this._doRequest({ ...config, method: 'GET' }, 'GET');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
_doRequest(config, method) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(config.url);
|
||||
const protocol = url.protocol === 'https:' ? https : http;
|
||||
|
||||
const options = {
|
||||
hostname: url.hostname,
|
||||
port: url.port || (url.protocol === 'https:' ? 443 : 80),
|
||||
path: url.pathname + url.search,
|
||||
method,
|
||||
timeout: config.timeout || 20000,
|
||||
headers: config.headers || {},
|
||||
rejectUnauthorized: false // Trust internal CA certs (.sami TLD)
|
||||
};
|
||||
|
||||
const req = protocol.request(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', chunk => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
const healthy = this.evaluateHealth(res.statusCode, data, config);
|
||||
|
||||
resolve({
|
||||
healthy,
|
||||
statusCode: res.statusCode,
|
||||
message: healthy ? 'Service is healthy' : 'Service check failed',
|
||||
details: {
|
||||
headers: res.headers,
|
||||
bodyLength: data.length
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('Health check timeout'));
|
||||
});
|
||||
|
||||
if (config.body) {
|
||||
req.write(JSON.stringify(config.body));
|
||||
}
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate if service is healthy based on response
|
||||
*/
|
||||
evaluateHealth(statusCode, body, config) {
|
||||
// Check status code
|
||||
const expectedCodes = config.expectedStatusCodes || [200, 201, 204, 301, 302, 303, 307, 308];
|
||||
if (!expectedCodes.includes(statusCode)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check response body if pattern specified
|
||||
if (config.expectedBodyPattern) {
|
||||
const regex = new RegExp(config.expectedBodyPattern);
|
||||
if (!regex.test(body)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Check response body contains expected text
|
||||
if (config.expectedBodyContains) {
|
||||
if (!body.includes(config.expectedBodyContains)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record service status
|
||||
*/
|
||||
recordStatus(serviceId, status) {
|
||||
// Update current status
|
||||
this.currentStatus.set(serviceId, status);
|
||||
|
||||
// Add to history
|
||||
if (!this.history[serviceId]) {
|
||||
this.history[serviceId] = [];
|
||||
}
|
||||
|
||||
this.history[serviceId].push(status);
|
||||
|
||||
// Emit status event
|
||||
this.emit('status-check', status);
|
||||
|
||||
// Save history periodically
|
||||
if (Math.random() < 0.05) { // 5% chance (every ~20 checks)
|
||||
this.saveHistory();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for incidents (downtime, slow response, etc.)
|
||||
*/
|
||||
checkForIncidents(serviceId, status, config) {
|
||||
const previous = this.currentStatus.get(serviceId);
|
||||
|
||||
// Check for status change (up -> down or down -> up)
|
||||
if (previous && previous.status !== status.status) {
|
||||
if (status.status === 'down') {
|
||||
this.createIncident(serviceId, 'outage', 'Service is down', status);
|
||||
} else if (status.status === 'up') {
|
||||
this.resolveIncident(serviceId, 'outage', status);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for slow response time
|
||||
const slowThreshold = config.slowResponseThreshold || 5000; // 5 seconds
|
||||
if (status.responseTime > slowThreshold) {
|
||||
this.createIncident(serviceId, 'slow-response',
|
||||
`Response time ${status.responseTime}ms exceeds threshold ${slowThreshold}ms`,
|
||||
status);
|
||||
}
|
||||
|
||||
// Check SLA violations
|
||||
const sla = config.sla;
|
||||
if (sla) {
|
||||
const uptime = this.calculateUptime(serviceId, sla.period || 24);
|
||||
if (uptime < sla.target) {
|
||||
this.createIncident(serviceId, 'sla-violation',
|
||||
`Uptime ${uptime.toFixed(2)}% below SLA target ${sla.target}%`,
|
||||
status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new incident
|
||||
*/
|
||||
createIncident(serviceId, type, message, status) {
|
||||
// Check if similar incident already exists
|
||||
const existing = this.incidents.find(i =>
|
||||
i.serviceId === serviceId &&
|
||||
i.type === type &&
|
||||
i.status === 'open'
|
||||
);
|
||||
|
||||
if (existing) {
|
||||
// Update existing incident
|
||||
existing.lastOccurrence = status.timestamp;
|
||||
existing.occurrences++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Create new incident
|
||||
const incident = {
|
||||
id: `incident-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
||||
serviceId,
|
||||
type,
|
||||
message,
|
||||
status: 'open',
|
||||
severity: this.calculateSeverity(type),
|
||||
createdAt: status.timestamp,
|
||||
lastOccurrence: status.timestamp,
|
||||
occurrences: 1,
|
||||
details: status
|
||||
};
|
||||
|
||||
this.incidents.push(incident);
|
||||
this.emit('incident-created', incident);
|
||||
|
||||
this.emit('log', 'info', `Incident created: ${incident.id} - ${message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an incident
|
||||
*/
|
||||
resolveIncident(serviceId, type, status) {
|
||||
const incident = this.incidents.find(i =>
|
||||
i.serviceId === serviceId &&
|
||||
i.type === type &&
|
||||
i.status === 'open'
|
||||
);
|
||||
|
||||
if (incident) {
|
||||
incident.status = 'resolved';
|
||||
incident.resolvedAt = status.timestamp;
|
||||
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
||||
|
||||
this.emit('incident-resolved', incident);
|
||||
this.emit('log', 'info', `Incident resolved: ${incident.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate incident severity
|
||||
*/
|
||||
calculateSeverity(type) {
|
||||
switch (type) {
|
||||
case 'outage':
|
||||
return 'critical';
|
||||
case 'sla-violation':
|
||||
return 'high';
|
||||
case 'slow-response':
|
||||
return 'medium';
|
||||
default:
|
||||
return 'low';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate uptime percentage for a service
|
||||
*/
|
||||
calculateUptime(serviceId, hours = 24) {
|
||||
const history = this.getServiceHistory(serviceId, hours);
|
||||
if (history.length === 0) return 100;
|
||||
|
||||
const upChecks = history.filter(h => h.status === 'up').length;
|
||||
return (upChecks / history.length) * 100;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate average response time
|
||||
*/
|
||||
calculateAverageResponseTime(serviceId, hours = 24) {
|
||||
const history = this.getServiceHistory(serviceId, hours);
|
||||
if (history.length === 0) return 0;
|
||||
|
||||
const total = history.reduce((sum, h) => sum + (h.responseTime || 0), 0);
|
||||
return total / history.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get service history for specified time period
|
||||
*/
|
||||
getServiceHistory(serviceId, hours = 24) {
|
||||
const cutoffTime = Date.now() - (hours * 60 * 60 * 1000);
|
||||
const history = this.history[serviceId] || [];
|
||||
|
||||
return history.filter(h =>
|
||||
new Date(h.timestamp).getTime() > cutoffTime
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current status for all services
|
||||
*/
|
||||
getCurrentStatus() {
|
||||
const result = {};
|
||||
|
||||
for (const [serviceId, status] of this.currentStatus.entries()) {
|
||||
const config = this.config.services[serviceId];
|
||||
const uptime24h = this.calculateUptime(serviceId, 24);
|
||||
const uptime7d = this.calculateUptime(serviceId, 168);
|
||||
const avgResponseTime = this.calculateAverageResponseTime(serviceId, 24);
|
||||
|
||||
result[serviceId] = {
|
||||
...status,
|
||||
name: config?.name || serviceId,
|
||||
uptime: {
|
||||
'24h': uptime24h,
|
||||
'7d': uptime7d
|
||||
},
|
||||
avgResponseTime,
|
||||
sla: config?.sla
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get service statistics
|
||||
*/
|
||||
getServiceStats(serviceId, hours = 24) {
|
||||
const history = this.getServiceHistory(serviceId, hours);
|
||||
if (history.length === 0) return null;
|
||||
|
||||
const upChecks = history.filter(h => h.status === 'up').length;
|
||||
const downChecks = history.length - upChecks;
|
||||
const responseTimes = history.map(h => h.responseTime || 0);
|
||||
|
||||
return {
|
||||
serviceId,
|
||||
period: `${hours}h`,
|
||||
totalChecks: history.length,
|
||||
upChecks,
|
||||
downChecks,
|
||||
uptime: (upChecks / history.length) * 100,
|
||||
responseTime: {
|
||||
avg: responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length,
|
||||
min: Math.min(...responseTimes),
|
||||
max: Math.max(...responseTimes),
|
||||
p95: this.calculatePercentile(responseTimes, 95),
|
||||
p99: this.calculatePercentile(responseTimes, 99)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate percentile
|
||||
*/
|
||||
calculatePercentile(values, percentile) {
|
||||
const sorted = values.slice().sort((a, b) => a - b);
|
||||
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
||||
return sorted[index] || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get open incidents
|
||||
*/
|
||||
getOpenIncidents() {
|
||||
return this.incidents.filter(i => i.status === 'open');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get incident history
|
||||
*/
|
||||
getIncidentHistory(limit = 50) {
|
||||
return this.incidents.slice(-limit).reverse();
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure health check for a service
|
||||
*/
|
||||
configureService(serviceId, config) {
|
||||
if (!this.config.services) {
|
||||
this.config.services = {};
|
||||
}
|
||||
|
||||
this.config.services[serviceId] = {
|
||||
enabled: config.enabled !== false,
|
||||
name: config.name || serviceId,
|
||||
url: config.url,
|
||||
method: config.method || 'GET',
|
||||
timeout: config.timeout || 20000,
|
||||
expectedStatusCodes: config.expectedStatusCodes || [200],
|
||||
expectedBodyPattern: config.expectedBodyPattern,
|
||||
expectedBodyContains: config.expectedBodyContains,
|
||||
slowResponseThreshold: config.slowResponseThreshold || 5000,
|
||||
sla: config.sla,
|
||||
headers: config.headers || {},
|
||||
body: config.body
|
||||
};
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove service configuration
|
||||
*/
|
||||
removeService(serviceId) {
|
||||
if (this.config.services) {
|
||||
delete this.config.services[serviceId];
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
this.currentStatus.delete(serviceId);
|
||||
delete this.history[serviceId];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup old history
|
||||
*/
|
||||
cleanupHistory() {
|
||||
const cutoffTime = Date.now() - (HISTORY_RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
||||
|
||||
for (const serviceId in this.history) {
|
||||
this.history[serviceId] = this.history[serviceId].filter(h =>
|
||||
new Date(h.timestamp).getTime() > cutoffTime
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration
|
||||
*/
|
||||
loadConfig() {
|
||||
try {
|
||||
if (fs.existsSync(HEALTH_CONFIG_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(HEALTH_CONFIG_FILE, 'utf8'));
|
||||
}
|
||||
} catch (error) {
|
||||
this.emit('log', 'error', `Error loading config: ${error.message}`);
|
||||
}
|
||||
return { services: {} };
|
||||
}
|
||||
|
||||
/**
|
||||
* Save configuration
|
||||
*/
|
||||
saveConfig() {
|
||||
try {
|
||||
fs.writeFileSync(HEALTH_CONFIG_FILE, JSON.stringify(this.config, null, 2));
|
||||
} catch (error) {
|
||||
this.emit('log', 'error', `Error saving config: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load history
|
||||
*/
|
||||
loadHistory() {
|
||||
try {
|
||||
if (fs.existsSync(HEALTH_HISTORY_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(HEALTH_HISTORY_FILE, 'utf8'));
|
||||
}
|
||||
} catch (error) {
|
||||
this.emit('log', 'error', `Error loading history: ${error.message}`);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Save history
|
||||
*/
|
||||
saveHistory() {
|
||||
try {
|
||||
fs.writeFileSync(HEALTH_HISTORY_FILE, JSON.stringify(this.history, null, 2));
|
||||
} catch (error) {
|
||||
this.emit('log', 'error', `Error saving history: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new HealthChecker();
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Simple metrics collector for DashCaddy API
|
||||
* Tracks request counts, durations, errors, and business metrics
|
||||
* No external dependencies — all in-memory
|
||||
*/
|
||||
|
||||
class Metrics {
|
||||
constructor() {
|
||||
this.startTime = Date.now();
|
||||
this.requests = {
|
||||
total: 0,
|
||||
byStatus: {},
|
||||
byMethod: {},
|
||||
byPath: {}
|
||||
};
|
||||
this.errors = {
|
||||
total: 0,
|
||||
byType: {}
|
||||
};
|
||||
this.business = {
|
||||
containersDeployed: 0,
|
||||
containersDeleted: 0,
|
||||
containerUpdates: 0,
|
||||
dnsRecordsCreated: 0,
|
||||
backupsCreated: 0,
|
||||
totpLogins: 0,
|
||||
siteAdded: 0,
|
||||
siteRemoved: 0,
|
||||
credentialRotations: 0
|
||||
};
|
||||
}
|
||||
|
||||
recordRequest(method, path, statusCode, durationMs) {
|
||||
this.requests.total++;
|
||||
this.requests.byStatus[statusCode] = (this.requests.byStatus[statusCode] || 0) + 1;
|
||||
this.requests.byMethod[method] = (this.requests.byMethod[method] || 0) + 1;
|
||||
|
||||
const normalized = this.normalizePath(path);
|
||||
if (!this.requests.byPath[normalized]) {
|
||||
this.requests.byPath[normalized] = { count: 0, totalDuration: 0 };
|
||||
}
|
||||
const entry = this.requests.byPath[normalized];
|
||||
entry.count++;
|
||||
entry.totalDuration += durationMs;
|
||||
}
|
||||
|
||||
recordError(errorType) {
|
||||
this.errors.total++;
|
||||
this.errors.byType[errorType] = (this.errors.byType[errorType] || 0) + 1;
|
||||
}
|
||||
|
||||
recordBusinessEvent(eventType) {
|
||||
if (eventType in this.business) {
|
||||
this.business[eventType]++;
|
||||
}
|
||||
}
|
||||
|
||||
normalizePath(p) {
|
||||
return p
|
||||
.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/:id')
|
||||
.replace(/\/[0-9a-f]{12,}/gi, '/:id')
|
||||
.replace(/\/\d+/g, '/:n');
|
||||
}
|
||||
|
||||
getSummary() {
|
||||
const uptimeMs = Date.now() - this.startTime;
|
||||
const uptimeSec = Math.floor(uptimeMs / 1000);
|
||||
|
||||
const topEndpoints = Object.entries(this.requests.byPath)
|
||||
.sort((a, b) => b[1].count - a[1].count)
|
||||
.slice(0, 15)
|
||||
.map(([path, s]) => ({ path, count: s.count, avgMs: Math.round(s.totalDuration / s.count) }));
|
||||
|
||||
return {
|
||||
uptime: { ms: uptimeMs, human: this.formatUptime(uptimeSec) },
|
||||
requests: {
|
||||
total: this.requests.total,
|
||||
perSecond: uptimeSec > 0 ? +(this.requests.total / uptimeSec).toFixed(2) : 0,
|
||||
byStatus: this.requests.byStatus,
|
||||
byMethod: this.requests.byMethod,
|
||||
topEndpoints
|
||||
},
|
||||
errors: {
|
||||
total: this.errors.total,
|
||||
rate: this.requests.total > 0 ? +((this.errors.total / this.requests.total) * 100).toFixed(2) : 0,
|
||||
byType: this.errors.byType
|
||||
},
|
||||
business: this.business,
|
||||
process: {
|
||||
memory: process.memoryUsage(),
|
||||
pid: process.pid,
|
||||
nodeVersion: process.version
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
formatUptime(sec) {
|
||||
const d = Math.floor(sec / 86400);
|
||||
const h = Math.floor((sec % 86400) / 3600);
|
||||
const m = Math.floor((sec % 3600) / 60);
|
||||
const s = sec % 60;
|
||||
if (d > 0) return `${d}d ${h}h ${m}m`;
|
||||
if (h > 0) return `${h}h ${m}m ${s}s`;
|
||||
if (m > 0) return `${m}m ${s}s`;
|
||||
return `${s}s`;
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.startTime = Date.now();
|
||||
this.requests = { total: 0, byStatus: {}, byMethod: {}, byPath: {} };
|
||||
this.errors = { total: 0, byType: {} };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new Metrics();
|
||||
@@ -0,0 +1,411 @@
|
||||
/**
|
||||
* SSL Certificate Monitor
|
||||
* Periodically checks SSL certificates on services with HTTPS URLs.
|
||||
* Alerts at 30, 14, and 7 days before expiry.
|
||||
*
|
||||
* @module ssl-monitor
|
||||
*/
|
||||
|
||||
const tls = require('tls');
|
||||
const EventEmitter = require('events');
|
||||
const path = require('path');
|
||||
const { readJsonFile, writeJsonFile } = require('../utilities/fs-helpers');
|
||||
const { resolveServiceUrl } = require('../utilities/url-resolver');
|
||||
|
||||
/** Default check interval: 1 hour */
|
||||
const DEFAULT_INTERVAL_MS = 3600000;
|
||||
|
||||
/** Alert thresholds in days */
|
||||
const THRESHOLDS = {
|
||||
WARNING: 30,
|
||||
URGENT: 14,
|
||||
CRITICAL: 7
|
||||
};
|
||||
|
||||
/** TLS connection timeout in milliseconds */
|
||||
const TLS_TIMEOUT_MS = 10000;
|
||||
|
||||
class SSLMonitor extends EventEmitter {
|
||||
/**
|
||||
* Create an SSLMonitor instance.
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.servicesStateManager - State manager for reading services
|
||||
* @param {Function} ctx.buildServiceUrl - URL builder helper
|
||||
* @param {Object} ctx.siteConfig - Site configuration
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {string} [ctx.SSL_CACHE_FILE] - Path to persist SSL cache
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
|
||||
/** @type {Map<string, Object>} hostname → last cert check result */
|
||||
this.certStatus = new Map();
|
||||
|
||||
/** @type {Map<string, number>} hostname → last notified threshold level */
|
||||
this.notifiedThresholds = new Map();
|
||||
|
||||
/** @type {Map<string, string>} hostname → service ID mapping */
|
||||
this.hostnameToServiceId = new Map();
|
||||
|
||||
/** @type {NodeJS.Timeout|null} */
|
||||
this.intervalHandle = null;
|
||||
|
||||
/** Current config */
|
||||
this.config = {
|
||||
enabled: true,
|
||||
intervalMs: DEFAULT_INTERVAL_MS
|
||||
};
|
||||
|
||||
/** Cache file path */
|
||||
this.cacheFile = ctx.SSL_CACHE_FILE ||
|
||||
path.join(path.dirname(ctx.SERVICES_FILE || './data'), 'ssl-cache.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the SSL certificate for a given hostname and port.
|
||||
* Connects via TLS with rejectUnauthorized: false to retrieve certificate info.
|
||||
*
|
||||
* @param {string} hostname - The hostname to check
|
||||
* @param {number} [port=443] - The port to connect to
|
||||
* @returns {Promise<Object>} Certificate information
|
||||
*/
|
||||
async checkCert(hostname, port = 443) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = tls.connect({
|
||||
host: hostname,
|
||||
port,
|
||||
rejectUnauthorized: false,
|
||||
servername: hostname,
|
||||
timeout: TLS_TIMEOUT_MS
|
||||
}, () => {
|
||||
try {
|
||||
const cert = socket.getPeerCertificate();
|
||||
|
||||
if (!cert || Object.keys(cert).length === 0) {
|
||||
socket.destroy();
|
||||
return reject(new Error(`No certificate returned for ${hostname}:${port}`));
|
||||
}
|
||||
|
||||
const validFrom = new Date(cert.valid_from);
|
||||
const validTo = new Date(cert.valid_to);
|
||||
const now = new Date();
|
||||
const msRemaining = validTo.getTime() - now.getTime();
|
||||
const daysRemaining = Math.ceil(msRemaining / (1000 * 60 * 60 * 24));
|
||||
|
||||
const result = {
|
||||
hostname,
|
||||
port,
|
||||
subject: cert.subject?.CN || cert.subject?.O || 'Unknown',
|
||||
issuer: cert.issuer?.CN || cert.issuer?.O || 'Unknown',
|
||||
validFrom: cert.valid_from,
|
||||
validTo: cert.valid_to,
|
||||
daysRemaining,
|
||||
fingerprint: cert.fingerprint || null,
|
||||
isExpiring: daysRemaining <= THRESHOLDS.WARNING,
|
||||
checkedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
socket.destroy();
|
||||
resolve(result);
|
||||
} catch (err) {
|
||||
socket.destroy();
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('error', (err) => {
|
||||
reject(new Error(`TLS connect error for ${hostname}:${port}: ${err.message}`));
|
||||
});
|
||||
|
||||
socket.setTimeout(TLS_TIMEOUT_MS, () => {
|
||||
socket.destroy(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||
reject(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check SSL certificates for all services that have HTTPS URLs.
|
||||
* Reads services from ctx.servicesStateManager, resolves URLs, and checks each HTTPS cert.
|
||||
*
|
||||
* @returns {Promise<Object>} Map of hostname → cert status
|
||||
*/
|
||||
async checkAll() {
|
||||
if (!this.config.enabled) {
|
||||
this.log.info('ssl-monitor', 'SSL monitoring is disabled, skipping check');
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
let servicesData;
|
||||
try {
|
||||
servicesData = await this.ctx.servicesStateManager.read();
|
||||
} catch (err) {
|
||||
this.log.error('ssl-monitor', 'Failed to read services', { error: err.message });
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
const services = Array.isArray(servicesData) ? servicesData : (servicesData.services || []);
|
||||
|
||||
for (const service of services) {
|
||||
const serviceId = service.id || service.name?.toLowerCase();
|
||||
if (!serviceId) continue;
|
||||
|
||||
try {
|
||||
const url = resolveServiceUrl(serviceId, service, this.ctx.siteConfig, this.ctx.buildServiceUrl);
|
||||
if (!url) continue;
|
||||
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== 'https:') continue;
|
||||
|
||||
const hostname = parsed.hostname;
|
||||
const port = parseInt(parsed.port) || 443;
|
||||
|
||||
// Map hostname back to service ID
|
||||
this.hostnameToServiceId.set(hostname, serviceId);
|
||||
|
||||
const result = await this.checkCert(hostname, port);
|
||||
|
||||
// Store result
|
||||
this.certStatus.set(hostname, result);
|
||||
|
||||
// Emit check event
|
||||
this.emit('cert-check', { serviceId, hostname, result });
|
||||
|
||||
// Check alert thresholds
|
||||
await this._checkAndNotify(hostname, result, serviceId);
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', `Failed to check cert for service ${serviceId}`, {
|
||||
error: err.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Persist results
|
||||
await this._saveCache();
|
||||
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic SSL certificate checking.
|
||||
*
|
||||
* @param {number} [intervalMs=3600000] - Check interval in milliseconds
|
||||
*/
|
||||
start(intervalMs) {
|
||||
if (intervalMs !== undefined) {
|
||||
this.config.intervalMs = intervalMs;
|
||||
}
|
||||
if (this.intervalHandle) {
|
||||
this.log.warn('ssl-monitor', 'SSL monitor is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.config.enabled = true;
|
||||
|
||||
// Load cached data
|
||||
this._loadCache().catch(err => {
|
||||
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||
});
|
||||
|
||||
// Initial check (non-blocking)
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Initial SSL check failed', { error: err.message });
|
||||
});
|
||||
|
||||
// Schedule periodic checks
|
||||
this.intervalHandle = setInterval(() => {
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||
});
|
||||
}, this.config.intervalMs);
|
||||
|
||||
this.log.info('ssl-monitor', 'SSL monitoring started', {
|
||||
intervalMs: this.config.intervalMs
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic SSL certificate checking.
|
||||
*/
|
||||
stop() {
|
||||
if (this.intervalHandle) {
|
||||
clearInterval(this.intervalHandle);
|
||||
this.intervalHandle = null;
|
||||
}
|
||||
this.config.enabled = false;
|
||||
this.log.info('ssl-monitor', 'SSL monitoring stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current SSL certificate status for all checked hostnames.
|
||||
*
|
||||
* @returns {Object} Map of hostname → cert status
|
||||
*/
|
||||
getStatus() {
|
||||
const status = {};
|
||||
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||
status[hostname] = { ...cert };
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the SSL certificate status for a specific service.
|
||||
*
|
||||
* @param {string} serviceId - The service ID to look up
|
||||
* @returns {Object|null} Certificate status or null if not found
|
||||
*/
|
||||
getServiceCertStatus(serviceId) {
|
||||
// Find hostname mapped to this service
|
||||
for (const [hostname, id] of this.hostnameToServiceId.entries()) {
|
||||
if (id === serviceId) {
|
||||
const cert = this.certStatus.get(hostname);
|
||||
return cert ? { ...cert, serviceId } : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current monitoring configuration.
|
||||
*
|
||||
* @returns {Object} Config with interval and enabled state
|
||||
*/
|
||||
getConfig() {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update monitoring configuration.
|
||||
*
|
||||
* @param {Object} updates - Config updates
|
||||
* @param {boolean} [updates.enabled] - Enable/disable monitoring
|
||||
* @param {number} [updates.intervalMs] - Check interval in milliseconds
|
||||
*/
|
||||
updateConfig(updates) {
|
||||
if (typeof updates.enabled === 'boolean') {
|
||||
this.config.enabled = updates.enabled;
|
||||
if (!updates.enabled && this.intervalHandle) {
|
||||
this.stop();
|
||||
}
|
||||
}
|
||||
if (typeof updates.intervalMs === 'number' && updates.intervalMs >= 60000) {
|
||||
this.config.intervalMs = updates.intervalMs;
|
||||
// Restart interval if running
|
||||
if (this.intervalHandle) {
|
||||
clearInterval(this.intervalHandle);
|
||||
this.intervalHandle = setInterval(() => {
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||
});
|
||||
}, this.config.intervalMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Private Methods =====
|
||||
|
||||
/**
|
||||
* Check alert thresholds and send notifications if thresholds are crossed.
|
||||
* Only sends one notification per threshold per hostname.
|
||||
*
|
||||
* @param {string} hostname
|
||||
* @param {Object} certResult
|
||||
* @param {string} serviceId
|
||||
*/
|
||||
async _checkAndNotify(hostname, certResult, serviceId) {
|
||||
const { daysRemaining } = certResult;
|
||||
const key = hostname;
|
||||
const lastNotified = this.notifiedThresholds.get(key) || Infinity;
|
||||
|
||||
let level = null;
|
||||
let eventType = null;
|
||||
let message = null;
|
||||
|
||||
if (daysRemaining <= THRESHOLDS.CRITICAL) {
|
||||
level = THRESHOLDS.CRITICAL;
|
||||
eventType = 'cert-critical';
|
||||
message = `🔒 CRITICAL: SSL certificate for ${hostname} expires in ${daysRemaining} days!`;
|
||||
} else if (daysRemaining <= THRESHOLDS.URGENT) {
|
||||
level = THRESHOLDS.URGENT;
|
||||
eventType = 'cert-expiring';
|
||||
message = `⚠️ URGENT: SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||
} else if (daysRemaining <= THRESHOLDS.WARNING) {
|
||||
level = THRESHOLDS.WARNING;
|
||||
eventType = 'cert-expiring';
|
||||
message = `⚠️ SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||
}
|
||||
|
||||
if (level !== null && level < lastNotified) {
|
||||
// New threshold crossed — send notification
|
||||
this.notifiedThresholds.set(key, level);
|
||||
this.emit(eventType, { hostname, serviceId, daysRemaining, level });
|
||||
|
||||
if (this.ctx.notification) {
|
||||
try {
|
||||
await this.ctx.notification.send('ssl-cert-expiry', {
|
||||
text: message,
|
||||
hostname,
|
||||
serviceId,
|
||||
daysRemaining,
|
||||
level,
|
||||
validTo: certResult.validTo
|
||||
}, level <= THRESHOLDS.CRITICAL ? 'error' : 'warning');
|
||||
} catch (err) {
|
||||
this.log.error('ssl-monitor', 'Failed to send SSL notification', { error: err.message });
|
||||
}
|
||||
}
|
||||
} else if (level === null) {
|
||||
// Cert is healthy — reset notification tracking
|
||||
this.notifiedThresholds.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist cert status cache to disk.
|
||||
*/
|
||||
async _saveCache() {
|
||||
try {
|
||||
const data = {
|
||||
lastChecked: new Date().toISOString(),
|
||||
certs: {},
|
||||
hostnameToServiceId: Object.fromEntries(this.hostnameToServiceId)
|
||||
};
|
||||
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||
data.certs[hostname] = cert;
|
||||
}
|
||||
await writeJsonFile(this.cacheFile, data);
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', 'Failed to save SSL cache', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load cert status cache from disk.
|
||||
*/
|
||||
async _loadCache() {
|
||||
try {
|
||||
const data = await readJsonFile(this.cacheFile, null);
|
||||
if (data && data.certs) {
|
||||
for (const [hostname, cert] of Object.entries(data.certs)) {
|
||||
this.certStatus.set(hostname, cert);
|
||||
}
|
||||
if (data.hostnameToServiceId) {
|
||||
for (const [hostname, serviceId] of Object.entries(data.hostnameToServiceId)) {
|
||||
this.hostnameToServiceId.set(hostname, serviceId);
|
||||
}
|
||||
}
|
||||
this.log.info('ssl-monitor', 'Loaded SSL cache', {
|
||||
certCount: this.certStatus.size
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SSLMonitor;
|
||||
Reference in New Issue
Block a user