diff --git a/dashcaddy-api/__tests__/health-checker-hysteresis.test.js b/dashcaddy-api/__tests__/health-checker-hysteresis.test.js index 49dd2ed..e4dcd42 100644 --- a/dashcaddy-api/__tests__/health-checker-hysteresis.test.js +++ b/dashcaddy-api/__tests__/health-checker-hysteresis.test.js @@ -22,11 +22,17 @@ process.env.HEALTH_DATA_DIR = tmpDir; process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json'); process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json'); -const { HealthChecker } = require('../src/monitoring/health-checker'); // Module exports a singleton instance, not a class — see module.exports in // src/monitoring/health-checker.js. The test creates fresh state by replacing // the relevant maps on the singleton in beforeEach. const healthCheckerSingleton = require('../src/monitoring/health-checker'); +const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD; +const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD; + +function restoreEnv(name, value) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} function makeUp(serviceId = 'svc1') { return { @@ -68,6 +74,15 @@ describe('DC-086: hysteresis on the dashboard badge', () => { hc = healthCheckerSingleton; }); + afterEach(() => { + restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold); + restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold); + }); + + afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + test('first probe (no prior state) emits', () => { hc.recordStatus('svc1', makeUp()); expect(emitSpy).toHaveBeenCalledTimes(1); @@ -88,6 +103,19 @@ describe('DC-086: hysteresis on the dashboard badge', () => { expect(hc.displayedStatus.get('svc1').status).toBe('up'); }); + test('up, down, up, down, down resets the first streak before flipping', () => { + hc.recordStatus('svc1', makeUp()); + hc.recordStatus('svc1', makeDown()); + hc.recordStatus('svc1', makeUp()); + hc.recordStatus('svc1', makeDown()); + expect(hc.displayedStatus.get('svc1').status).toBe('up'); + expect(emitSpy).toHaveBeenCalledTimes(1); + + hc.recordStatus('svc1', makeDown()); + expect(hc.displayedStatus.get('svc1').status).toBe('down'); + expect(emitSpy).toHaveBeenCalledTimes(2); + }); + test('two consecutive "down" probes flip the badge to red', () => { hc.recordStatus('svc1', makeUp()); // baseline: green hc.recordStatus('svc1', makeDown()); // blip #1 — keep green (counter=1) @@ -120,10 +148,28 @@ describe('DC-086: hysteresis on the dashboard badge', () => { }); test('getCurrentStatus returns the displayed status, not the raw probe', () => { - hc.recordStatus('svc1', makeUp()); - hc.recordStatus('svc1', makeDown()); // raw=down, displayed=up + const displayedUp = makeUp(); + displayedUp.timestamp = '2026-08-22T09:59:00.000Z'; + displayedUp.statusCode = 200; + displayedUp.message = 'healthy'; + displayedUp.details = { source: 'accepted-up' }; + hc.recordStatus('svc1', displayedUp); + const latestRaw = makeDown(); + latestRaw.timestamp = '2026-08-22T10:00:00.000Z'; + latestRaw.responseTime = 987; + latestRaw.statusCode = 500; + latestRaw.message = 'failed probe'; + latestRaw.error = 'upstream failure'; + latestRaw.details = { source: 'suppressed-down' }; + hc.recordStatus('svc1', latestRaw); // raw=down, displayed=up const out = hc.getCurrentStatus(); expect(out['svc1'].status).toBe('up'); // shown to API consumers + expect(out['svc1'].timestamp).toBe(displayedUp.timestamp); + expect(out['svc1'].statusCode).toBe(200); + expect(out['svc1'].message).toBe('healthy'); + expect(out['svc1'].error).toBeUndefined(); + expect(out['svc1'].details).toEqual({ source: 'accepted-up' }); + expect(hc.currentStatus.get('svc1')).toBe(latestRaw); }); test('a long steady-green run produces exactly ONE emit (no per-probe spam)', () => { @@ -142,7 +188,6 @@ describe('DC-086: hysteresis on the dashboard badge', () => { }); test('DOWN_THRESHOLD env var is honored', () => { - const origDown = process.env.HEALTH_DOWN_THRESHOLD; process.env.HEALTH_DOWN_THRESHOLD = '3'; jest.resetModules(); const HC2Module = require('../src/monitoring/health-checker'); @@ -164,7 +209,89 @@ describe('DC-086: hysteresis on the dashboard badge', () => { hc2.recordStatus('svc1', makeDown()); // 3 — flip expect(spy).toHaveBeenCalledTimes(2); expect(hc2.displayedStatus.get('svc1').status).toBe('down'); - if (origDown === undefined) delete process.env.HEALTH_DOWN_THRESHOLD; - else process.env.HEALTH_DOWN_THRESHOLD = origDown; }); -}); \ No newline at end of file + + test.each(['not-a-number', '0', '-2', '1.5'])('malformed DOWN_THRESHOLD %s falls back to 2', value => { + process.env.HEALTH_DOWN_THRESHOLD = value; + jest.resetModules(); + const hc2 = require('../src/monitoring/health-checker'); + hc2.displayedStatus = new Map(); + hc2.consecutiveSinceChange = new Map(); + hc2.currentStatus = new Map(); + hc2.history = {}; + hc2.removeAllListeners('status-check'); + const spy = jest.fn(); + hc2.on('status-check', spy); + hc2.recordStatus('svc1', makeUp()); + hc2.recordStatus('svc1', makeDown()); + expect(spy).toHaveBeenCalledTimes(1); + hc2.recordStatus('svc1', makeDown()); + expect(spy).toHaveBeenCalledTimes(2); + }); + + test('UP_THRESHOLD env var greater than 1 is honored', () => { + process.env.HEALTH_UP_THRESHOLD = '2'; + jest.resetModules(); + const hc2 = require('../src/monitoring/health-checker'); + hc2.displayedStatus = new Map(); + hc2.consecutiveSinceChange = new Map(); + hc2.currentStatus = new Map(); + hc2.history = {}; + hc2.removeAllListeners('status-check'); + const spy = jest.fn(); + hc2.on('status-check', spy); + hc2.recordStatus('svc1', makeDown()); + hc2.recordStatus('svc1', makeUp()); + expect(spy).toHaveBeenCalledTimes(1); + expect(hc2.displayedStatus.get('svc1').status).toBe('down'); + hc2.recordStatus('svc1', makeUp()); + expect(spy).toHaveBeenCalledTimes(2); + expect(hc2.displayedStatus.get('svc1').status).toBe('up'); + }); + + test.each(['not-a-number', '0', '-2', '1.5'])('malformed UP_THRESHOLD %s falls back to 1', value => { + process.env.HEALTH_UP_THRESHOLD = value; + jest.resetModules(); + const hc2 = require('../src/monitoring/health-checker'); + hc2.displayedStatus = new Map(); + hc2.consecutiveSinceChange = new Map(); + hc2.currentStatus = new Map(); + hc2.history = {}; + hc2.removeAllListeners('status-check'); + const spy = jest.fn(); + hc2.on('status-check', spy); + hc2.recordStatus('svc1', makeDown()); + hc2.recordStatus('svc1', makeUp()); + expect(spy).toHaveBeenCalledTimes(2); + expect(hc2.displayedStatus.get('svc1').status).toBe('up'); + }); + + test('removeService clears hysteresis state before the same ID is re-added', () => { + hc.config.services.svc1 = { name: 'Service 1' }; + hc.recordStatus('svc1', makeUp()); + hc.recordStatus('svc1', makeDown()); + expect(hc.displayedStatus.has('svc1')).toBe(true); + expect(hc.consecutiveSinceChange.get('svc1')).toBe(1); + hc.consecutiveFailures.set('svc1', 3); + const timer = setTimeout(() => {}, 60_000); + hc.serviceTimers.set('svc1', timer); + + hc.saveConfig = jest.fn(); + hc.removeService('svc1'); + + expect(hc.displayedStatus.has('svc1')).toBe(false); + expect(hc.consecutiveSinceChange.has('svc1')).toBe(false); + expect(hc.currentStatus.has('svc1')).toBe(false); + expect(hc.consecutiveFailures.has('svc1')).toBe(false); + expect(hc.serviceTimers.has('svc1')).toBe(false); + + hc.config.services.svc1 = { name: 'Service 1 re-added' }; + const emitSpyAfterReAdd = jest.fn(); + hc.on('status-check', emitSpyAfterReAdd); + hc.recordStatus('svc1', makeDown()); + + expect(emitSpyAfterReAdd).toHaveBeenCalledTimes(1); + expect(hc.displayedStatus.get('svc1').status).toBe('down'); + expect(hc.consecutiveSinceChange.has('svc1')).toBe(false); + }); +}); diff --git a/dashcaddy-api/__tests__/health-checker.test.js b/dashcaddy-api/__tests__/health-checker.test.js index 55f2e74..64b6ca1 100644 --- a/dashcaddy-api/__tests__/health-checker.test.js +++ b/dashcaddy-api/__tests__/health-checker.test.js @@ -203,6 +203,48 @@ describe('HealthChecker', () => { expect(result.error).toBe('ECONNREFUSED'); }); + it('opens and resolves an outage incident across real checkService transitions', async () => { + healthChecker._doRequest = jest.fn() + .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} }) + .mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} }) + .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} }); + + const config = { url: 'http://test.local' }; + await healthChecker.checkService('svc1', config); + await healthChecker.checkService('svc1', config); + + expect(healthChecker.incidents).toHaveLength(1); + expect(healthChecker.incidents[0]).toMatchObject({ + serviceId: 'svc1', + type: 'outage', + status: 'open' + }); + + await healthChecker.checkService('svc1', config); + expect(healthChecker.incidents[0].status).toBe('resolved'); + expect(healthChecker.incidents[0].resolvedAt).toBeDefined(); + }); + + it('does not resurrect state when an in-flight probe resolves after removal', async () => { + let resolveProbe; + healthChecker.config.services.svc1 = { url: 'http://test.local' }; + healthChecker._doRequest = jest.fn(() => new Promise(resolve => { + resolveProbe = resolve; + })); + + const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1); + healthChecker.saveConfig = jest.fn(); + healthChecker.removeService('svc1'); + resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} }); + await pending; + + expect(healthChecker.currentStatus.has('svc1')).toBe(false); + expect(healthChecker.displayedStatus.has('svc1')).toBe(false); + expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false); + expect(healthChecker.history.svc1).toBeUndefined(); + expect(healthChecker.incidents).toEqual([]); + }); + it('increments consecutive failures on error', async () => { healthChecker._doRequest = jest.fn().mockRejectedValue(new Error('fail')); diff --git a/dashcaddy-api/src/monitoring/health-checker.js b/dashcaddy-api/src/monitoring/health-checker.js index 44ec2f7..572d342 100644 --- a/dashcaddy-api/src/monitoring/health-checker.js +++ b/dashcaddy-api/src/monitoring/health-checker.js @@ -43,8 +43,15 @@ const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || // - UP_THRESHOLD = N consecutive "up" probes before the badge flips back to green // Single probe flips to green on purpose — false-positive-green is much less // painful than perpetual-red (operators notice red, ignore green). -const DOWN_THRESHOLD = Math.max(1, parseInt(process.env.HEALTH_DOWN_THRESHOLD || '2', 10)); -const UP_THRESHOLD = Math.max(1, parseInt(process.env.HEALTH_UP_THRESHOLD || '1', 10)); +function readPositiveIntEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw === '') return fallback; + const value = Number(raw); + return Number.isSafeInteger(value) && value >= 1 ? value : fallback; +} + +const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2); +const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1); class HealthChecker extends EventEmitter { constructor() { @@ -63,6 +70,8 @@ class HealthChecker extends EventEmitter { this.checkInterval = null; this.consecutiveFailures = new Map(); // serviceId -> failure count this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff + // Invalidate probe completions that race with removal/reconfiguration. + this.serviceGenerations = new Map(); // serviceId -> configuration generation } /** @@ -135,6 +144,7 @@ class HealthChecker extends EventEmitter { */ async checkService(serviceId, config) { const startTime = Date.now(); + const generation = this.serviceGenerations.get(serviceId) || 0; try { const result = await this.performHealthCheck(config); @@ -150,6 +160,10 @@ class HealthChecker extends EventEmitter { details: result.details }; + if ((this.serviceGenerations.get(serviceId) || 0) !== generation) { + return status; + } + // Track consecutive failures for exponential backoff if (result.healthy) { this.consecutiveFailures.delete(serviceId); @@ -157,8 +171,9 @@ class HealthChecker extends EventEmitter { this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1); } + const previousStatus = this.currentStatus.get(serviceId); this.recordStatus(serviceId, status); - this.checkForIncidents(serviceId, status, config); + this.checkForIncidents(serviceId, status, config, previousStatus); return status; } catch (error) { @@ -175,8 +190,13 @@ class HealthChecker extends EventEmitter { error: error.message }; + if ((this.serviceGenerations.get(serviceId) || 0) !== generation) { + return status; + } + + const previousStatus = this.currentStatus.get(serviceId); this.recordStatus(serviceId, status); - this.checkForIncidents(serviceId, status, config); + this.checkForIncidents(serviceId, status, config, previousStatus); return status; } @@ -319,7 +339,7 @@ class HealthChecker extends EventEmitter { // a brief blip doesn't accumulate against the displayed state. if (rawStatus.status === previousStatus) { this.consecutiveSinceChange.set(serviceId, 0); - return currentDisplayed; + return rawStatus; } // Probe disagrees with displayed. Bump the streak counter — this counts @@ -334,6 +354,9 @@ class HealthChecker extends EventEmitter { // with the displayed "up" state. if (previousStatus === 'up' && next < DOWN_THRESHOLD) { this.consecutiveSinceChange.set(serviceId, next); + // Keep the last internally-consistent displayed snapshot. Mixing the + // raw failure metadata with status="up" would expose contradictory + // API data (for example statusCode=500 on an "up" service). return currentDisplayed; } // Threshold met (or already down) — flip to red. @@ -402,8 +425,7 @@ class HealthChecker extends EventEmitter { /** * Check for incidents (downtime, slow response, etc.) */ - checkForIncidents(serviceId, status, config) { - const previous = this.currentStatus.get(serviceId); + checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId)) { // Check for status change (up -> down or down -> up) if (previous && previous.status !== status.status) { @@ -638,6 +660,7 @@ class HealthChecker extends EventEmitter { this.config.services = {}; } + this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1); this.config.services[serviceId] = { enabled: config.enabled !== false, name: config.name || serviceId, @@ -660,12 +683,19 @@ class HealthChecker extends EventEmitter { * Remove service configuration */ removeService(serviceId) { + this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1); if (this.config.services) { delete this.config.services[serviceId]; this.saveConfig(); } this.currentStatus.delete(serviceId); + this.displayedStatus.delete(serviceId); + this.consecutiveSinceChange.delete(serviceId); + this.consecutiveFailures.delete(serviceId); + const timer = this.serviceTimers.get(serviceId); + if (timer) clearTimeout(timer); + this.serviceTimers.delete(serviceId); delete this.history[serviceId]; }