Hermes
88f1d4a414
checkForIncidents compared raw probe transitions while the dashboard badge (DC-086) follows post-hysteresis displayed status. A single raw down blip between two ups opened AND resolved a critical outage incident; a suppressed up blip during a real outage resolved it early. Incidents now open/resolve on displayed-vs-displayed transitions; previousDisplayed=null keeps legacy raw semantics for direct callers. 6 new parity tests + legacy checkService test moved to a 4-probe chain. Suite 2616/2616 (110). Verdict: urn:ump:yc5rdlnmnmhch5audc5fifgbt6d7moi2stqfs5vidsbh6x6zkvgq
864 lines
30 KiB
JavaScript
864 lines
30 KiB
JavaScript
/**
|
|
* Health Check Dashboard Module
|
|
* Monitors service health, response times, and uptime
|
|
* Provides SLA tracking and incident management
|
|
*/
|
|
|
|
const https = require('https');
|
|
const http = require('http');
|
|
const crypto = require('crypto');
|
|
const EventEmitter = require('events');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const paths = require('../../platform-paths');
|
|
|
|
// Persist health config + history alongside the other state files (services.json,
|
|
// config.json) rather than next to the source. In a container that data dir is the
|
|
// mounted /app/data volume, so uptime history survives container recreates/updates;
|
|
// previously these defaulted to __dirname (unmounted /app) and every recreate wiped
|
|
// the accumulated history, blanking the dashboard uptime bars. Explicit env vars
|
|
// still override.
|
|
const HEALTH_DATA_DIR = process.env.HEALTH_DATA_DIR || path.dirname(paths.configFile);
|
|
const HEALTH_CONFIG_FILE = process.env.HEALTH_CONFIG_FILE || path.join(HEALTH_DATA_DIR, 'health-config.json');
|
|
const HEALTH_HISTORY_FILE = process.env.HEALTH_HISTORY_FILE || path.join(HEALTH_DATA_DIR, 'health-history.json');
|
|
|
|
// Legacy locations (next to the source) used before the data-dir default. Read these
|
|
// once on first load if the new files are absent, so upgrading installs migrate their
|
|
// accumulated history/config instead of starting empty. The next save() rewrites to
|
|
// the new location.
|
|
const LEGACY_HEALTH_CONFIG_FILE = path.join(__dirname, 'health-config.json');
|
|
const LEGACY_HEALTH_HISTORY_FILE = path.join(__dirname, 'health-history.json');
|
|
const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10); // 30 seconds
|
|
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
|
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
|
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
|
// DC-088: how long a removal tombstone outlives the removal itself. Only needs
|
|
// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
|
|
// swept by cleanupHistory so removed services cannot accumulate map entries.
|
|
const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
|
|
|
|
// DC-086: hysteresis thresholds for badge display.
|
|
// The raw probe result can flap on a single transient blip (Caddy reload,
|
|
// container CPU steal, network hiccup, mid-flight TLS handshake). Showing
|
|
// every probe result as-is to the dashboard creates the "perpetual flicker"
|
|
// UX. Asymmetric thresholds: going red is slow (don't false-alarm), going
|
|
// green is fast (don't keep showing red after recovery).
|
|
// - DOWN_THRESHOLD = N consecutive "down" probes before the badge flips to red
|
|
// - UP_THRESHOLD = N consecutive "up" probes before the badge flips back to green
|
|
// Single probe flips to green on purpose — false-positive-green is much less
|
|
// painful than perpetual-red (operators notice red, ignore green).
|
|
function readPositiveIntEnv(name, fallback) {
|
|
const raw = process.env[name];
|
|
if (raw === undefined || raw === '') return fallback;
|
|
const value = Number(raw);
|
|
return Number.isSafeInteger(value) && value >= 1 ? value : fallback;
|
|
}
|
|
|
|
const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2);
|
|
const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1);
|
|
|
|
class HealthChecker extends EventEmitter {
|
|
constructor() {
|
|
super();
|
|
this.config = this.loadConfig();
|
|
this.history = this.loadHistory();
|
|
this.currentStatus = new Map();
|
|
// DC-086: the status the dashboard SHOULD display (post-hysteresis).
|
|
// Distinct from currentStatus, which is the latest raw probe result.
|
|
this.displayedStatus = new Map();
|
|
// DC-086: counter of consecutive healthy/unhealthy probes since the
|
|
// last displayed-status change. Reset to 0 whenever displayed status flips.
|
|
this.consecutiveSinceChange = new Map();
|
|
this.incidents = [];
|
|
this.checking = false;
|
|
this.checkInterval = null;
|
|
this.consecutiveFailures = new Map(); // serviceId -> failure count
|
|
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
|
// Invalidate probe completions that race with removal/reconfiguration.
|
|
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
|
// DC-088: monotonically increasing sequence so generation numbers can never
|
|
// repeat across remove -> re-add cycles (prevents ABA on the stale check).
|
|
this.generationSeq = 0;
|
|
// DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
|
|
// serviceGenerations means the service is (re)configured; a tombstone with a
|
|
// HIGHER generation than the captured one marks the capture as stale. Entry
|
|
// is deleted when the service is removed, so the live map cannot leak.
|
|
this.removedGenerations = new Map();
|
|
}
|
|
|
|
/**
|
|
* Start health checking
|
|
*/
|
|
start() {
|
|
if (this.checking) return;
|
|
|
|
this.checking = true;
|
|
|
|
// Initial check
|
|
this.checkAll();
|
|
|
|
// Schedule periodic checks
|
|
this.checkInterval = setInterval(() => this.checkAll(), CHECK_INTERVAL);
|
|
}
|
|
|
|
/**
|
|
* Stop health checking
|
|
*/
|
|
stop() {
|
|
if (!this.checking) return;
|
|
|
|
this.checking = false;
|
|
|
|
if (this.checkInterval) {
|
|
clearInterval(this.checkInterval);
|
|
this.checkInterval = null;
|
|
}
|
|
|
|
// Clear per-service backoff timers
|
|
for (const timer of this.serviceTimers.values()) {
|
|
clearTimeout(timer);
|
|
}
|
|
this.serviceTimers.clear();
|
|
}
|
|
|
|
/**
|
|
* Get the backoff interval for a service based on consecutive failures.
|
|
* Doubles the interval for each failure, capped at MAX_CHECK_INTERVAL.
|
|
*/
|
|
getBackoffInterval(serviceId) {
|
|
const failures = this.consecutiveFailures.get(serviceId) || 0;
|
|
if (failures === 0) return CHECK_INTERVAL;
|
|
return Math.min(CHECK_INTERVAL * Math.pow(2, failures), MAX_CHECK_INTERVAL);
|
|
}
|
|
|
|
/**
|
|
* Check all configured services
|
|
*/
|
|
async checkAll() {
|
|
const services = Object.entries(this.config.services || {});
|
|
|
|
for (const [serviceId, config] of services) {
|
|
if (config.enabled !== false) {
|
|
try {
|
|
await this.checkService(serviceId, config);
|
|
} catch (error) {
|
|
// Error logged via checkForIncidents
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cleanup old history
|
|
this.cleanupHistory();
|
|
}
|
|
|
|
/**
|
|
* DC-088: true when a probe's captured generation no longer matches the
|
|
* service's current configuration state. A live serviceGenerations entry
|
|
* must match exactly. With no live entry the service was never configured
|
|
* in this process (disk-loaded / direct callers) — stale only if a removal
|
|
* tombstone with a HIGHER generation exists.
|
|
*/
|
|
_isStaleCapture(serviceId, generation) {
|
|
if (this.serviceGenerations.has(serviceId)) {
|
|
return this.serviceGenerations.get(serviceId) !== generation;
|
|
}
|
|
const tomb = this.removedGenerations.get(serviceId);
|
|
return Boolean(tomb && tomb.generation > generation);
|
|
}
|
|
|
|
/**
|
|
* Check a single service
|
|
*/
|
|
async checkService(serviceId, config) {
|
|
const startTime = Date.now();
|
|
const generation = this.serviceGenerations.get(serviceId) || 0;
|
|
|
|
try {
|
|
const result = await this.performHealthCheck(config);
|
|
const responseTime = Date.now() - startTime;
|
|
|
|
const status = {
|
|
serviceId,
|
|
timestamp: new Date().toISOString(),
|
|
status: result.healthy ? 'up' : 'down',
|
|
responseTime,
|
|
statusCode: result.statusCode,
|
|
message: result.message,
|
|
details: result.details
|
|
};
|
|
|
|
if (this._isStaleCapture(serviceId, generation)) {
|
|
return status;
|
|
}
|
|
|
|
// Track consecutive failures for exponential backoff
|
|
if (result.healthy) {
|
|
this.consecutiveFailures.delete(serviceId);
|
|
} else {
|
|
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
|
}
|
|
|
|
const previousStatus = this.currentStatus.get(serviceId);
|
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
|
this.recordStatus(serviceId, status);
|
|
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
|
|
|
return status;
|
|
} catch (error) {
|
|
const responseTime = Date.now() - startTime;
|
|
|
|
const status = {
|
|
serviceId,
|
|
timestamp: new Date().toISOString(),
|
|
status: 'down',
|
|
responseTime,
|
|
error: error.message
|
|
};
|
|
|
|
if (this._isStaleCapture(serviceId, generation)) {
|
|
return status;
|
|
}
|
|
|
|
// Increment failure count for backoff — only after the result is known
|
|
// to be non-stale, so a removed service cannot re-create map entries.
|
|
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
|
|
|
const previousStatus = this.currentStatus.get(serviceId);
|
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
|
this.recordStatus(serviceId, status);
|
|
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
|
|
|
return status;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Perform actual health check
|
|
*/
|
|
async performHealthCheck(config) {
|
|
const result = await this._doRequest(config, config.method || 'GET');
|
|
// Fall back to GET if HEAD is not supported
|
|
if ((result.statusCode === 501 || result.statusCode === 405) && (config.method || '').toUpperCase() === 'HEAD') {
|
|
return this._doRequest({ ...config, method: 'GET' }, 'GET');
|
|
}
|
|
return result;
|
|
}
|
|
|
|
_doRequest(config, method) {
|
|
return new Promise((resolve, reject) => {
|
|
const url = new URL(config.url);
|
|
const protocol = url.protocol === 'https:' ? https : http;
|
|
|
|
// Merge user-supplied headers with the health-check marker. Caddy on
|
|
// *.sami uses `forward_auth` for every non-API path and the auth gate
|
|
// returns 401 for HEAD/GET without a session — without this marker the
|
|
// probe never reaches the upstream service, and the authLimiter (20 req
|
|
// / 15 min) on /auth/* would also rate-limit us after 20 probes. The
|
|
// marker lets the Caddy snippet bypass forward_auth for probes that
|
|
// originate from the local container network (see /etc/caddy/Caddyfile
|
|
// `(dashcaddy_auth)` block).
|
|
const headers = {
|
|
...(config.headers || {}),
|
|
'X-DashCaddy-HealthCheck': '1'
|
|
};
|
|
|
|
const options = {
|
|
hostname: url.hostname,
|
|
port: url.port || (url.protocol === 'https:' ? 443 : 80),
|
|
path: url.pathname + url.search,
|
|
method,
|
|
timeout: config.timeout || 20000,
|
|
headers,
|
|
rejectUnauthorized: false // Trust internal CA certs (.sami TLD)
|
|
};
|
|
|
|
const req = protocol.request(options, (res) => {
|
|
let data = '';
|
|
|
|
res.on('data', chunk => {
|
|
data += chunk;
|
|
});
|
|
|
|
res.on('end', () => {
|
|
const healthy = this.evaluateHealth(res.statusCode, data, config);
|
|
|
|
resolve({
|
|
healthy,
|
|
statusCode: res.statusCode,
|
|
message: healthy ? 'Service is healthy' : 'Service check failed',
|
|
details: {
|
|
headers: res.headers ? { server: res.headers.server } : undefined, // Compact: disk explosion fix
|
|
bodyLength: data.length
|
|
}
|
|
});
|
|
});
|
|
});
|
|
|
|
req.on('error', (error) => {
|
|
reject(error);
|
|
});
|
|
|
|
req.on('timeout', () => {
|
|
req.destroy();
|
|
reject(new Error('Health check timeout'));
|
|
});
|
|
|
|
if (config.body) {
|
|
req.write(JSON.stringify(config.body));
|
|
}
|
|
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Evaluate if service is healthy based on response
|
|
*/
|
|
evaluateHealth(statusCode, body, config) {
|
|
// Check status code. Default expected codes include the usual 2xx/3xx
|
|
// plus 401/403 (auth-walled UIs that still prove the service is up) and
|
|
// 429 (rate-limited upstream — we hit the service, the service answered;
|
|
// failing the check just because we're being throttled is wrong).
|
|
const expectedCodes = config.expectedStatusCodes || [200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429];
|
|
if (!expectedCodes.includes(statusCode)) {
|
|
return false;
|
|
}
|
|
|
|
// Check response body if pattern specified
|
|
if (config.expectedBodyPattern) {
|
|
const regex = new RegExp(config.expectedBodyPattern);
|
|
if (!regex.test(body)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Check response body contains expected text
|
|
if (config.expectedBodyContains) {
|
|
if (!body.includes(config.expectedBodyContains)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Compute the displayed status for a service given the latest raw probe
|
|
* result. Applies asymmetric hysteresis:
|
|
* - Going DOWN: requires DOWN_THRESHOLD (default 2) consecutive "down"
|
|
* probes since the last display-state change. A single blip keeps the
|
|
* badge green.
|
|
* - Going UP: requires UP_THRESHOLD (default 1) consecutive "up" probes.
|
|
* Any single "up" after a down streak flips back to green so the badge
|
|
* doesn't linger red after the service has recovered.
|
|
*
|
|
* Returns the displayed status object (same shape as the raw status) so
|
|
* recordStatus can use it both for the displayed map and as the broadcast
|
|
* payload when the displayed status actually changes.
|
|
*/
|
|
_computeDisplayedStatus(serviceId, rawStatus) {
|
|
const currentDisplayed = this.displayedStatus.get(serviceId);
|
|
const previousStatus = currentDisplayed ? currentDisplayed.status : null;
|
|
|
|
// If no prior state, accept the raw probe as-is (first-check bootstrap).
|
|
if (!previousStatus) {
|
|
return rawStatus;
|
|
}
|
|
|
|
// Probe agrees with current displayed → no change, reset the counter so
|
|
// a brief blip doesn't accumulate against the displayed state.
|
|
if (rawStatus.status === previousStatus) {
|
|
this.consecutiveSinceChange.set(serviceId, 0);
|
|
return rawStatus;
|
|
}
|
|
|
|
// Probe disagrees with displayed. Bump the streak counter — this counts
|
|
// CONSECUTIVE probes that disagree with what's shown, regardless of
|
|
// whether the raw value itself changed between probes. That's what
|
|
// makes "down, down" flip after threshold but "down, up, down" not flip.
|
|
const prev = this.consecutiveSinceChange.get(serviceId) || 0;
|
|
const next = prev + 1;
|
|
|
|
if (rawStatus.status === 'down') {
|
|
// Going DOWN: need DOWN_THRESHOLD consecutive probes that disagree
|
|
// with the displayed "up" state.
|
|
if (previousStatus === 'up' && next < DOWN_THRESHOLD) {
|
|
this.consecutiveSinceChange.set(serviceId, next);
|
|
// Keep the last internally-consistent displayed snapshot. Mixing the
|
|
// raw failure metadata with status="up" would expose contradictory
|
|
// API data (for example statusCode=500 on an "up" service).
|
|
return currentDisplayed;
|
|
}
|
|
// Threshold met (or already down) — flip to red.
|
|
this.consecutiveSinceChange.set(serviceId, 0);
|
|
return rawStatus;
|
|
}
|
|
|
|
// rawStatus.status === 'up' (must be — the equal-to-displayed case above
|
|
// already returned). Going UP after a down streak: need UP_THRESHOLD.
|
|
if (previousStatus === 'down' && next < UP_THRESHOLD) {
|
|
this.consecutiveSinceChange.set(serviceId, next);
|
|
return currentDisplayed;
|
|
}
|
|
this.consecutiveSinceChange.set(serviceId, 0);
|
|
return rawStatus;
|
|
}
|
|
|
|
/**
|
|
* Record service status
|
|
*
|
|
* DC-086: history + consecutiveFailures are updated for EVERY probe
|
|
* (operators want full probe history for postmortems). The dashboard's
|
|
* `status-check` event is only emitted when the DISPLAYED status changes,
|
|
* so the badge stops re-rendering on every probe.
|
|
*/
|
|
recordStatus(serviceId, status) {
|
|
// Update current (raw) status — used by checkForIncidents and history.
|
|
this.currentStatus.set(serviceId, status);
|
|
|
|
// Add raw probe to history (full fidelity — operators rely on this).
|
|
if (!this.history[serviceId]) {
|
|
this.history[serviceId] = [];
|
|
}
|
|
|
|
this.history[serviceId].push(status);
|
|
|
|
// Cap entries to prevent unbounded growth (disk explosion fix)
|
|
if (this.history[serviceId].length > MAX_ENTRIES_PER_SERVICE) {
|
|
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
|
}
|
|
|
|
// Compute the post-hysteresis displayed status; only emit when it changes.
|
|
// _computeDisplayedStatus compares the raw probe against the DISPLAYED
|
|
// status (not the previous raw status), so the "consecutive since
|
|
// change" counter doesn't depend on the order of writes here.
|
|
const displayed = this._computeDisplayedStatus(serviceId, status);
|
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
|
const displayChanged =
|
|
!previousDisplayed || previousDisplayed.status !== displayed.status;
|
|
|
|
this.displayedStatus.set(serviceId, displayed);
|
|
|
|
if (displayChanged) {
|
|
// Emit with the displayed status so the dashboard renders the same
|
|
// state the hysteresis just decided. The raw probe result is still
|
|
// in `history` and `currentStatus` for anyone who wants it.
|
|
this.emit('status-check', displayed);
|
|
}
|
|
|
|
// Save history periodically
|
|
if (Math.random() < 0.05) { // 5% chance (every ~20 checks)
|
|
this.saveHistory();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check for incidents (downtime, slow response, etc.)
|
|
*/
|
|
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
|
|
|
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
|
// the same signal that flips the dashboard badge. A single raw "down"
|
|
// blip that hysteresis suppresses must not open a critical outage
|
|
// incident (and a suppressed blip must not resolve a real one). When the
|
|
// caller supplies the pre-probe displayed state (checkService always
|
|
// does), transitions are evaluated displayed-vs-displayed using the
|
|
// post-recordStatus state in this.displayedStatus. Direct callers with
|
|
// no hysteresis state (previousDisplayed === null) keep the legacy
|
|
// raw-probe transition semantics.
|
|
if (previousDisplayed) {
|
|
const displayed = this.displayedStatus.get(serviceId);
|
|
if (displayed && displayed.status !== previousDisplayed.status) {
|
|
if (displayed.status === 'down') {
|
|
this.createIncident(serviceId, 'outage', 'Service is down', displayed);
|
|
} else if (displayed.status === 'up') {
|
|
this.resolveIncident(serviceId, 'outage', displayed);
|
|
}
|
|
}
|
|
} else if (previous && previous.status !== status.status) {
|
|
if (status.status === 'down') {
|
|
this.createIncident(serviceId, 'outage', 'Service is down', status);
|
|
} else if (status.status === 'up') {
|
|
this.resolveIncident(serviceId, 'outage', status);
|
|
}
|
|
}
|
|
|
|
// Check for slow response time
|
|
const slowThreshold = config.slowResponseThreshold || 5000; // 5 seconds
|
|
if (status.responseTime > slowThreshold) {
|
|
this.createIncident(serviceId, 'slow-response',
|
|
`Response time ${status.responseTime}ms exceeds threshold ${slowThreshold}ms`,
|
|
status);
|
|
}
|
|
|
|
// Check SLA violations
|
|
const sla = config.sla;
|
|
if (sla) {
|
|
const uptime = this.calculateUptime(serviceId, sla.period || 24);
|
|
if (uptime < sla.target) {
|
|
this.createIncident(serviceId, 'sla-violation',
|
|
`Uptime ${uptime.toFixed(2)}% below SLA target ${sla.target}%`,
|
|
status);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a new incident
|
|
*/
|
|
createIncident(serviceId, type, message, status) {
|
|
// Check if similar incident already exists
|
|
const existing = this.incidents.find(i =>
|
|
i.serviceId === serviceId &&
|
|
i.type === type &&
|
|
i.status === 'open'
|
|
);
|
|
|
|
if (existing) {
|
|
// Update existing incident
|
|
existing.lastOccurrence = status.timestamp;
|
|
existing.occurrences++;
|
|
return;
|
|
}
|
|
|
|
// Create new incident
|
|
const incident = {
|
|
id: `incident-${crypto.randomUUID()}`,
|
|
serviceId,
|
|
type,
|
|
message,
|
|
status: 'open',
|
|
severity: this.calculateSeverity(type),
|
|
createdAt: status.timestamp,
|
|
lastOccurrence: status.timestamp,
|
|
occurrences: 1,
|
|
details: status
|
|
};
|
|
|
|
this.incidents.push(incident);
|
|
this.emit('incident-created', incident);
|
|
|
|
this.emit('log', 'info', `Incident created: ${incident.id} - ${message}`);
|
|
}
|
|
|
|
/**
|
|
* Resolve an incident
|
|
*/
|
|
resolveIncident(serviceId, type, status) {
|
|
const incident = this.incidents.find(i =>
|
|
i.serviceId === serviceId &&
|
|
i.type === type &&
|
|
i.status === 'open'
|
|
);
|
|
|
|
if (incident) {
|
|
incident.status = 'resolved';
|
|
incident.resolvedAt = status.timestamp;
|
|
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
|
|
|
this.emit('incident-resolved', incident);
|
|
this.emit('log', 'info', `Incident resolved: ${incident.id}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate incident severity
|
|
*/
|
|
calculateSeverity(type) {
|
|
switch (type) {
|
|
case 'outage':
|
|
return 'critical';
|
|
case 'sla-violation':
|
|
return 'high';
|
|
case 'slow-response':
|
|
return 'medium';
|
|
default:
|
|
return 'low';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate uptime percentage for a service
|
|
*/
|
|
calculateUptime(serviceId, hours = 24) {
|
|
const history = this.getServiceHistory(serviceId, hours);
|
|
if (history.length === 0) return 100;
|
|
|
|
const upChecks = history.filter(h => h.status === 'up').length;
|
|
return (upChecks / history.length) * 100;
|
|
}
|
|
|
|
/**
|
|
* Calculate average response time
|
|
*/
|
|
calculateAverageResponseTime(serviceId, hours = 24) {
|
|
const history = this.getServiceHistory(serviceId, hours);
|
|
if (history.length === 0) return 0;
|
|
|
|
const total = history.reduce((sum, h) => sum + (h.responseTime || 0), 0);
|
|
return total / history.length;
|
|
}
|
|
|
|
/**
|
|
* Get service history for specified time period
|
|
*/
|
|
getServiceHistory(serviceId, hours = 24) {
|
|
const cutoffTime = Date.now() - (hours * 60 * 60 * 1000);
|
|
const history = this.history[serviceId] || [];
|
|
|
|
return history.filter(h =>
|
|
new Date(h.timestamp).getTime() > cutoffTime
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get current status for all services.
|
|
*
|
|
* DC-086: returns the DISPLAYED status (post-hysteresis), not the latest
|
|
* raw probe. A page reload should show the same badge state the live
|
|
* SSE stream is currently showing — otherwise an operator who reloads
|
|
* the page after a single blip sees red even though the hysteresis kept
|
|
* the badge green for them.
|
|
*/
|
|
getCurrentStatus() {
|
|
const result = {};
|
|
|
|
for (const [serviceId, rawStatus] of this.currentStatus.entries()) {
|
|
const config = this.config.services[serviceId];
|
|
const uptime24h = this.calculateUptime(serviceId, 24);
|
|
const uptime7d = this.calculateUptime(serviceId, 168);
|
|
const avgResponseTime = this.calculateAverageResponseTime(serviceId, 24);
|
|
|
|
// Prefer the displayed status if we've already computed one; fall back
|
|
// to the raw probe on the very first call (before recordStatus has run).
|
|
const displayed = this.displayedStatus.get(serviceId) || rawStatus;
|
|
|
|
result[serviceId] = {
|
|
...displayed,
|
|
name: config?.name || serviceId,
|
|
uptime: {
|
|
'24h': uptime24h,
|
|
'7d': uptime7d
|
|
},
|
|
avgResponseTime,
|
|
sla: config?.sla
|
|
};
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Get service statistics
|
|
*/
|
|
getServiceStats(serviceId, hours = 24) {
|
|
const history = this.getServiceHistory(serviceId, hours);
|
|
if (history.length === 0) return null;
|
|
|
|
const upChecks = history.filter(h => h.status === 'up').length;
|
|
const downChecks = history.length - upChecks;
|
|
const responseTimes = history.map(h => h.responseTime || 0);
|
|
|
|
return {
|
|
serviceId,
|
|
period: `${hours}h`,
|
|
totalChecks: history.length,
|
|
upChecks,
|
|
downChecks,
|
|
uptime: (upChecks / history.length) * 100,
|
|
responseTime: {
|
|
avg: responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length,
|
|
min: Math.min(...responseTimes),
|
|
max: Math.max(...responseTimes),
|
|
p95: this.calculatePercentile(responseTimes, 95),
|
|
p99: this.calculatePercentile(responseTimes, 99)
|
|
}
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Calculate percentile
|
|
*/
|
|
calculatePercentile(values, percentile) {
|
|
const sorted = values.slice().sort((a, b) => a - b);
|
|
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
|
return sorted[index] || 0;
|
|
}
|
|
|
|
/**
|
|
* Get open incidents
|
|
*/
|
|
getOpenIncidents() {
|
|
return this.incidents.filter(i => i.status === 'open');
|
|
}
|
|
|
|
/**
|
|
* Get incident history
|
|
*/
|
|
getIncidentHistory(limit = 50) {
|
|
return this.incidents.slice(-limit).reverse();
|
|
}
|
|
|
|
/**
|
|
* Configure health check for a service
|
|
*/
|
|
configureService(serviceId, config) {
|
|
if (!this.config.services) {
|
|
this.config.services = {};
|
|
}
|
|
|
|
// DC-088: monotonic instance-wide sequence — a re-added service can never
|
|
// recycle a previous generation number, and any older in-flight capture is
|
|
// invalidated by definition.
|
|
this.generationSeq += 1;
|
|
this.serviceGenerations.set(serviceId, this.generationSeq);
|
|
// Re-configuration supersedes any prior removal tombstone.
|
|
this.removedGenerations.delete(serviceId);
|
|
this.config.services[serviceId] = {
|
|
enabled: config.enabled !== false,
|
|
name: config.name || serviceId,
|
|
url: config.url,
|
|
method: config.method || 'GET',
|
|
timeout: config.timeout || 20000,
|
|
expectedStatusCodes: config.expectedStatusCodes || [200],
|
|
expectedBodyPattern: config.expectedBodyPattern,
|
|
expectedBodyContains: config.expectedBodyContains,
|
|
slowResponseThreshold: config.slowResponseThreshold || 5000,
|
|
sla: config.sla,
|
|
headers: config.headers || {},
|
|
body: config.body
|
|
};
|
|
|
|
this.saveConfig();
|
|
}
|
|
|
|
/**
|
|
* Remove service configuration
|
|
*/
|
|
removeService(serviceId) {
|
|
// DC-088: tombstone the captured generation instead of leaking an entry.
|
|
// The live map entry is deleted; an in-flight probe captured BEFORE this
|
|
// point sees no live entry but a higher tombstone generation, so it is
|
|
// discarded. configureService clears the tombstone on re-add.
|
|
this.generationSeq += 1;
|
|
this.serviceGenerations.delete(serviceId);
|
|
this.removedGenerations.set(serviceId, {
|
|
generation: this.generationSeq,
|
|
removedAt: Date.now()
|
|
});
|
|
if (this.config.services) {
|
|
delete this.config.services[serviceId];
|
|
this.saveConfig();
|
|
}
|
|
|
|
// DC-088: open incidents for a removed service must not linger forever.
|
|
// Close them through the same resolve path a recovery would, annotated so
|
|
// history shows why (dashboard renders resolved incidents green + duration).
|
|
for (const incident of this.incidents) {
|
|
if (incident.serviceId === serviceId && incident.status === 'open') {
|
|
incident.status = 'resolved';
|
|
incident.resolvedAt = new Date().toISOString();
|
|
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
|
incident.resolvedBy = 'service-removed';
|
|
this.emit('incident-resolved', incident);
|
|
this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
|
|
}
|
|
}
|
|
|
|
this.currentStatus.delete(serviceId);
|
|
this.displayedStatus.delete(serviceId);
|
|
this.consecutiveSinceChange.delete(serviceId);
|
|
this.consecutiveFailures.delete(serviceId);
|
|
const timer = this.serviceTimers.get(serviceId);
|
|
if (timer) clearTimeout(timer);
|
|
this.serviceTimers.delete(serviceId);
|
|
delete this.history[serviceId];
|
|
}
|
|
|
|
/**
|
|
* Cleanup old history
|
|
*/
|
|
cleanupHistory() {
|
|
const cutoffTime = Date.now() - (HISTORY_RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
|
|
|
for (const serviceId in this.history) {
|
|
this.history[serviceId] = this.history[serviceId].filter(h =>
|
|
new Date(h.timestamp).getTime() > cutoffTime
|
|
);
|
|
// Also cap total entries per service
|
|
if (this.history[serviceId].length > MAX_ENTRIES_PER_SERVICE) {
|
|
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
|
}
|
|
}
|
|
|
|
// DC-088: sweep expired removal tombstones. After the TTL no probe that
|
|
// captured a pre-removal generation can still be in flight (timeout is
|
|
// bounded by performHealthCheck), so the tombstone has done its job.
|
|
if (this.removedGenerations.size > 0) {
|
|
const now = Date.now();
|
|
for (const [serviceId, tomb] of this.removedGenerations) {
|
|
if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
|
|
this.removedGenerations.delete(serviceId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load configuration
|
|
*/
|
|
loadConfig() {
|
|
try {
|
|
const file = fs.existsSync(HEALTH_CONFIG_FILE) ? HEALTH_CONFIG_FILE
|
|
: (HEALTH_CONFIG_FILE !== LEGACY_HEALTH_CONFIG_FILE && fs.existsSync(LEGACY_HEALTH_CONFIG_FILE) ? LEGACY_HEALTH_CONFIG_FILE : null);
|
|
if (file) {
|
|
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
}
|
|
} catch (error) {
|
|
this.emit('log', 'error', `Error loading config: ${error.message}`);
|
|
}
|
|
return { services: {} };
|
|
}
|
|
|
|
/**
|
|
* Save configuration
|
|
*/
|
|
saveConfig() {
|
|
try {
|
|
fs.writeFileSync(HEALTH_CONFIG_FILE, JSON.stringify(this.config, null, 2));
|
|
} catch (error) {
|
|
this.emit('log', 'error', `Error saving config: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load history
|
|
*/
|
|
loadHistory() {
|
|
try {
|
|
const file = fs.existsSync(HEALTH_HISTORY_FILE) ? HEALTH_HISTORY_FILE
|
|
: (HEALTH_HISTORY_FILE !== LEGACY_HEALTH_HISTORY_FILE && fs.existsSync(LEGACY_HEALTH_HISTORY_FILE) ? LEGACY_HEALTH_HISTORY_FILE : null);
|
|
if (file) {
|
|
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
}
|
|
} catch (error) {
|
|
this.emit('log', 'error', `Error loading history: ${error.message}`);
|
|
}
|
|
return {};
|
|
}
|
|
|
|
/**
|
|
* Save history
|
|
*/
|
|
saveHistory() {
|
|
try {
|
|
fs.writeFileSync(HEALTH_HISTORY_FILE, JSON.stringify(this.history)); // Compact JSON (no pretty-print) to reduce file size
|
|
} catch (error) {
|
|
this.emit('log', 'error', `Error saving history: ${error.message}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Export singleton instance
|
|
module.exports = new HealthChecker();
|