Files
dashcaddy/dashcaddy-api/__tests__/health-checker-hysteresis.test.js
T
Hermes be021588c7 DC-086 service-status flicker fix — asymmetric hysteresis
Dashboard badges perpetually flip green/red for a few seconds at a time,
never stable. Root cause: health-checker emitted 'status-check' on every
probe (every 30s) and dashboard-ws forwarded every one as 'status-change'
to the browser with no diff; live-events.js then unconditionally called
setBadge(). A single transient 5xx (Caddy reload, container restart, TLS
handshake blip) flipped the badge and the next green probe flipped back.

Fix: _computeDisplayedStatus applies asymmetric hysteresis — DOWN_THRESHOLD
(default 2, env-tunable HEALTH_DOWN_THRESHOLD) consecutive probes that
disagree with the displayed 'up' state flip to red; UP_THRESHOLD (default
1, HEALTH_UP_THRESHOLD) flips back to green. History and consecutiveFailures
still record every raw probe so postmortem analysis is unchanged. Only the
SSE broadcast is filtered. getCurrentStatus now returns the displayed
status so a page reload shows the same badge as the live stream.

10 new tests cover first-emit, same-status-dedup, the actual flicker bug
(one-down-then-up keeps green), two-down flips red, one-up recovers fast,
long-steady-green produces exactly one emit, and env-var tuning. All 63
existing health-checker tests still pass. Full suite: 2484/2484.
2026-08-20 04:46:17 -07:00

170 lines
6.8 KiB
JavaScript

/**
* Tests for DC-086: asymmetric hysteresis on the dashboard service badge.
*
* - First probe always emits (no prior state).
* - Same-status probe does NOT re-emit (dedup against repeated green).
* - One "down" then back to "up" keeps the badge green (no flicker).
* - Two consecutive "down" probes flip the badge to red.
* - One "up" after a down streak flips back to green (fast recovery).
* - History retains every raw probe even when no emit happens.
* - getCurrentStatus returns displayed status, not raw.
*/
'use strict';
const path = require('path');
const fs = require('fs');
const os = require('os');
// Use an isolated data dir so test history doesn't pollute the real one.
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dashcaddy-hyst-'));
process.env.HEALTH_DATA_DIR = tmpDir;
process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
const { HealthChecker } = require('../src/monitoring/health-checker');
// Module exports a singleton instance, not a class — see module.exports in
// src/monitoring/health-checker.js. The test creates fresh state by replacing
// the relevant maps on the singleton in beforeEach.
const healthCheckerSingleton = require('../src/monitoring/health-checker');
function makeUp(serviceId = 'svc1') {
return {
serviceId,
timestamp: new Date().toISOString(),
status: 'up',
responseTime: 50,
statusCode: 200,
message: 'Service is healthy',
details: { headers: {}, bodyLength: 12 }
};
}
function makeDown(serviceId = 'svc1') {
return {
serviceId,
timestamp: new Date().toISOString(),
status: 'down',
responseTime: 50,
statusCode: 500,
message: 'fail',
details: { headers: {}, bodyLength: 0 }
};
}
describe('DC-086: hysteresis on the dashboard badge', () => {
let hc;
let emitSpy;
beforeEach(() => {
// Reset the singleton's per-test state so each case starts clean.
healthCheckerSingleton.displayedStatus = new Map();
healthCheckerSingleton.consecutiveSinceChange = new Map();
healthCheckerSingleton.currentStatus = new Map();
healthCheckerSingleton.history = {};
healthCheckerSingleton.removeAllListeners('status-check');
emitSpy = jest.fn();
healthCheckerSingleton.on('status-check', emitSpy);
hc = healthCheckerSingleton;
});
test('first probe (no prior state) emits', () => {
hc.recordStatus('svc1', makeUp());
expect(emitSpy).toHaveBeenCalledTimes(1);
expect(emitSpy.mock.calls[0][0].status).toBe('up');
});
test('second probe with same status does NOT re-emit', () => {
hc.recordStatus('svc1', makeUp());
hc.recordStatus('svc1', makeUp());
expect(emitSpy).toHaveBeenCalledTimes(1);
});
test('one "down" then "up" keeps the badge green (the flicker bug)', () => {
hc.recordStatus('svc1', makeUp()); // baseline: green, emit 1
hc.recordStatus('svc1', makeDown()); // one blip — keep green, no emit
hc.recordStatus('svc1', makeUp()); // recovered — still green, no emit
expect(emitSpy).toHaveBeenCalledTimes(1);
expect(hc.displayedStatus.get('svc1').status).toBe('up');
});
test('two consecutive "down" probes flip the badge to red', () => {
hc.recordStatus('svc1', makeUp()); // baseline: green
hc.recordStatus('svc1', makeDown()); // blip #1 — keep green (counter=1)
hc.recordStatus('svc1', makeDown()); // blip #2 — flip red (counter=2 >= DOWN_THRESHOLD)
expect(emitSpy).toHaveBeenCalledTimes(2);
expect(emitSpy.mock.calls[1][0].status).toBe('down');
expect(hc.displayedStatus.get('svc1').status).toBe('down');
});
test('one "up" after a down streak flips back to green (fast recovery)', () => {
hc.recordStatus('svc1', makeUp());
hc.recordStatus('svc1', makeDown());
hc.recordStatus('svc1', makeDown()); // now red
expect(hc.displayedStatus.get('svc1').status).toBe('down');
hc.recordStatus('svc1', makeUp()); // first green — flip back
expect(emitSpy).toHaveBeenCalledTimes(3);
expect(emitSpy.mock.calls[2][0].status).toBe('up');
expect(hc.displayedStatus.get('svc1').status).toBe('up');
});
test('history retains every raw probe even when no emit happens', () => {
hc.recordStatus('svc1', makeUp());
hc.recordStatus('svc1', makeDown()); // blip, no emit
hc.recordStatus('svc1', makeUp()); // recovery, no emit
expect(hc.history['svc1'].length).toBe(3);
expect(hc.history['svc1'][0].status).toBe('up');
expect(hc.history['svc1'][1].status).toBe('down');
expect(hc.history['svc1'][2].status).toBe('up');
});
test('getCurrentStatus returns the displayed status, not the raw probe', () => {
hc.recordStatus('svc1', makeUp());
hc.recordStatus('svc1', makeDown()); // raw=down, displayed=up
const out = hc.getCurrentStatus();
expect(out['svc1'].status).toBe('up'); // shown to API consumers
});
test('a long steady-green run produces exactly ONE emit (no per-probe spam)', () => {
for (let i = 0; i < 50; i++) hc.recordStatus('svc1', makeUp());
expect(emitSpy).toHaveBeenCalledTimes(1);
});
test('a long steady-green-then-steady-red transition: 1 emit (up), 1 emit (red)', () => {
for (let i = 0; i < 10; i++) hc.recordStatus('svc1', makeUp());
expect(emitSpy).toHaveBeenCalledTimes(1);
hc.recordStatus('svc1', makeDown());
hc.recordStatus('svc1', makeDown()); // flips to red
expect(emitSpy).toHaveBeenCalledTimes(2);
for (let i = 0; i < 10; i++) hc.recordStatus('svc1', makeDown());
expect(emitSpy).toHaveBeenCalledTimes(2); // no further broadcasts
});
test('DOWN_THRESHOLD env var is honored', () => {
const origDown = process.env.HEALTH_DOWN_THRESHOLD;
process.env.HEALTH_DOWN_THRESHOLD = '3';
jest.resetModules();
const HC2Module = require('../src/monitoring/health-checker');
// Module is a singleton with DOWN_THRESHOLD captured at module load —
// resetModules gives us a fresh module-level instance with the new env.
const hc2 = HC2Module;
hc2.displayedStatus = new Map();
hc2.consecutiveSinceChange = new Map();
hc2.currentStatus = new Map();
hc2.history = {};
hc2.removeAllListeners('status-check');
const spy = jest.fn();
hc2.on('status-check', spy);
hc2.recordStatus('svc1', makeUp());
hc2.recordStatus('svc1', makeDown()); // 1
hc2.recordStatus('svc1', makeDown()); // 2 — still green (need 3)
expect(spy).toHaveBeenCalledTimes(1);
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
hc2.recordStatus('svc1', makeDown()); // 3 — flip
expect(spy).toHaveBeenCalledTimes(2);
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
if (origDown === undefined) delete process.env.HEALTH_DOWN_THRESHOLD;
else process.env.HEALTH_DOWN_THRESHOLD = origDown;
});
});