Hermes
628bbe32f6
Round-2 folds the judge-round fixes into DC-086: - serviceGenerations map: checkService captures the config generation at entry and re-validates it before ANY state write (success + error paths). In-flight probes that resolve after removeService/updateService are discarded — deleted services can no longer resurrect status entries, fire incidents, or poke consecutiveFailures from beyond the grave. - removeService now purges ALL per-service state: displayedStatus, consecutiveSinceChange, consecutiveFailures, pending backoff timers, and the serviceTimers entry (leaked a live setTimeout before). - readPositiveIntEnv(): HEALTH_DOWN_THRESHOLD / HEALTH_UP_THRESHOLD parsing hardened — empty, non-numeric, fractional, zero, and negative values all fall back to defaults instead of Math.max(1, NaN)=NaN. - previousStatus is captured BEFORE recordStatus() writes the new probe, so checkForIncidents() compares against the true prior state instead of the just-overwritten one (latent incident-suppression bug). - Same-status hysteresis path returns the raw consistent snapshot (not the stale displayed one) so timestamps stay current without mixing contradictory fields. - Tests: +14 (86 total across the two suites). New coverage: streak reset on agreement, malformed env fallbacks (each.of not-a-number/0/ -2/1.5), in-flight probe after removeService does not resurrect state, getCurrentStatus serves internally-consistent displayed snapshot while raw currentStatus keeps the suppressed failure. Full suite 2601/2601. Judge: GLM-5.3 cold-read via delegate_task (deleg_c9fd5900 task-0), grade A round 1, zero blocking issues. Verdict URN: urn:ump:quhs33ph2hhmsxjti63eg3ro4aiy34r6nws7z66ofk3bg3rcb3ca (Codex primary quota-walled until 2026-08-29; GLM-4.6 direct 401; stand-in chain per codex-as-judge SKILL.md, Sami 2026-08-17.)
298 lines
12 KiB
JavaScript
298 lines
12 KiB
JavaScript
/**
|
|
* Tests for DC-086: asymmetric hysteresis on the dashboard service badge.
|
|
*
|
|
* - First probe always emits (no prior state).
|
|
* - Same-status probe does NOT re-emit (dedup against repeated green).
|
|
* - One "down" then back to "up" keeps the badge green (no flicker).
|
|
* - Two consecutive "down" probes flip the badge to red.
|
|
* - One "up" after a down streak flips back to green (fast recovery).
|
|
* - History retains every raw probe even when no emit happens.
|
|
* - getCurrentStatus returns displayed status, not raw.
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
|
|
// Use an isolated data dir so test history doesn't pollute the real one.
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dashcaddy-hyst-'));
|
|
process.env.HEALTH_DATA_DIR = tmpDir;
|
|
process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
|
|
process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
|
|
|
|
// Module exports a singleton instance, not a class — see module.exports in
|
|
// src/monitoring/health-checker.js. The test creates fresh state by replacing
|
|
// the relevant maps on the singleton in beforeEach.
|
|
const healthCheckerSingleton = require('../src/monitoring/health-checker');
|
|
const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD;
|
|
const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD;
|
|
|
|
function restoreEnv(name, value) {
|
|
if (value === undefined) delete process.env[name];
|
|
else process.env[name] = value;
|
|
}
|
|
|
|
function makeUp(serviceId = 'svc1') {
|
|
return {
|
|
serviceId,
|
|
timestamp: new Date().toISOString(),
|
|
status: 'up',
|
|
responseTime: 50,
|
|
statusCode: 200,
|
|
message: 'Service is healthy',
|
|
details: { headers: {}, bodyLength: 12 }
|
|
};
|
|
}
|
|
|
|
function makeDown(serviceId = 'svc1') {
|
|
return {
|
|
serviceId,
|
|
timestamp: new Date().toISOString(),
|
|
status: 'down',
|
|
responseTime: 50,
|
|
statusCode: 500,
|
|
message: 'fail',
|
|
details: { headers: {}, bodyLength: 0 }
|
|
};
|
|
}
|
|
|
|
describe('DC-086: hysteresis on the dashboard badge', () => {
|
|
let hc;
|
|
let emitSpy;
|
|
|
|
beforeEach(() => {
|
|
// Reset the singleton's per-test state so each case starts clean.
|
|
healthCheckerSingleton.displayedStatus = new Map();
|
|
healthCheckerSingleton.consecutiveSinceChange = new Map();
|
|
healthCheckerSingleton.currentStatus = new Map();
|
|
healthCheckerSingleton.history = {};
|
|
healthCheckerSingleton.removeAllListeners('status-check');
|
|
emitSpy = jest.fn();
|
|
healthCheckerSingleton.on('status-check', emitSpy);
|
|
hc = healthCheckerSingleton;
|
|
});
|
|
|
|
afterEach(() => {
|
|
restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold);
|
|
restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold);
|
|
});
|
|
|
|
afterAll(() => {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
test('first probe (no prior state) emits', () => {
|
|
hc.recordStatus('svc1', makeUp());
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
expect(emitSpy.mock.calls[0][0].status).toBe('up');
|
|
});
|
|
|
|
test('second probe with same status does NOT re-emit', () => {
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeUp());
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
test('one "down" then "up" keeps the badge green (the flicker bug)', () => {
|
|
hc.recordStatus('svc1', makeUp()); // baseline: green, emit 1
|
|
hc.recordStatus('svc1', makeDown()); // one blip — keep green, no emit
|
|
hc.recordStatus('svc1', makeUp()); // recovered — still green, no emit
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('up');
|
|
});
|
|
|
|
test('up, down, up, down, down resets the first streak before flipping', () => {
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeDown());
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeDown());
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('up');
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
|
|
hc.recordStatus('svc1', makeDown());
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('down');
|
|
expect(emitSpy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
test('two consecutive "down" probes flip the badge to red', () => {
|
|
hc.recordStatus('svc1', makeUp()); // baseline: green
|
|
hc.recordStatus('svc1', makeDown()); // blip #1 — keep green (counter=1)
|
|
hc.recordStatus('svc1', makeDown()); // blip #2 — flip red (counter=2 >= DOWN_THRESHOLD)
|
|
expect(emitSpy).toHaveBeenCalledTimes(2);
|
|
expect(emitSpy.mock.calls[1][0].status).toBe('down');
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('down');
|
|
});
|
|
|
|
test('one "up" after a down streak flips back to green (fast recovery)', () => {
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeDown());
|
|
hc.recordStatus('svc1', makeDown()); // now red
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('down');
|
|
|
|
hc.recordStatus('svc1', makeUp()); // first green — flip back
|
|
expect(emitSpy).toHaveBeenCalledTimes(3);
|
|
expect(emitSpy.mock.calls[2][0].status).toBe('up');
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('up');
|
|
});
|
|
|
|
test('history retains every raw probe even when no emit happens', () => {
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeDown()); // blip, no emit
|
|
hc.recordStatus('svc1', makeUp()); // recovery, no emit
|
|
expect(hc.history['svc1'].length).toBe(3);
|
|
expect(hc.history['svc1'][0].status).toBe('up');
|
|
expect(hc.history['svc1'][1].status).toBe('down');
|
|
expect(hc.history['svc1'][2].status).toBe('up');
|
|
});
|
|
|
|
test('getCurrentStatus returns the displayed status, not the raw probe', () => {
|
|
const displayedUp = makeUp();
|
|
displayedUp.timestamp = '2026-08-22T09:59:00.000Z';
|
|
displayedUp.statusCode = 200;
|
|
displayedUp.message = 'healthy';
|
|
displayedUp.details = { source: 'accepted-up' };
|
|
hc.recordStatus('svc1', displayedUp);
|
|
const latestRaw = makeDown();
|
|
latestRaw.timestamp = '2026-08-22T10:00:00.000Z';
|
|
latestRaw.responseTime = 987;
|
|
latestRaw.statusCode = 500;
|
|
latestRaw.message = 'failed probe';
|
|
latestRaw.error = 'upstream failure';
|
|
latestRaw.details = { source: 'suppressed-down' };
|
|
hc.recordStatus('svc1', latestRaw); // raw=down, displayed=up
|
|
const out = hc.getCurrentStatus();
|
|
expect(out['svc1'].status).toBe('up'); // shown to API consumers
|
|
expect(out['svc1'].timestamp).toBe(displayedUp.timestamp);
|
|
expect(out['svc1'].statusCode).toBe(200);
|
|
expect(out['svc1'].message).toBe('healthy');
|
|
expect(out['svc1'].error).toBeUndefined();
|
|
expect(out['svc1'].details).toEqual({ source: 'accepted-up' });
|
|
expect(hc.currentStatus.get('svc1')).toBe(latestRaw);
|
|
});
|
|
|
|
test('a long steady-green run produces exactly ONE emit (no per-probe spam)', () => {
|
|
for (let i = 0; i < 50; i++) hc.recordStatus('svc1', makeUp());
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
test('a long steady-green-then-steady-red transition: 1 emit (up), 1 emit (red)', () => {
|
|
for (let i = 0; i < 10; i++) hc.recordStatus('svc1', makeUp());
|
|
expect(emitSpy).toHaveBeenCalledTimes(1);
|
|
hc.recordStatus('svc1', makeDown());
|
|
hc.recordStatus('svc1', makeDown()); // flips to red
|
|
expect(emitSpy).toHaveBeenCalledTimes(2);
|
|
for (let i = 0; i < 10; i++) hc.recordStatus('svc1', makeDown());
|
|
expect(emitSpy).toHaveBeenCalledTimes(2); // no further broadcasts
|
|
});
|
|
|
|
test('DOWN_THRESHOLD env var is honored', () => {
|
|
process.env.HEALTH_DOWN_THRESHOLD = '3';
|
|
jest.resetModules();
|
|
const HC2Module = require('../src/monitoring/health-checker');
|
|
// Module is a singleton with DOWN_THRESHOLD captured at module load —
|
|
// resetModules gives us a fresh module-level instance with the new env.
|
|
const hc2 = HC2Module;
|
|
hc2.displayedStatus = new Map();
|
|
hc2.consecutiveSinceChange = new Map();
|
|
hc2.currentStatus = new Map();
|
|
hc2.history = {};
|
|
hc2.removeAllListeners('status-check');
|
|
const spy = jest.fn();
|
|
hc2.on('status-check', spy);
|
|
hc2.recordStatus('svc1', makeUp());
|
|
hc2.recordStatus('svc1', makeDown()); // 1
|
|
hc2.recordStatus('svc1', makeDown()); // 2 — still green (need 3)
|
|
expect(spy).toHaveBeenCalledTimes(1);
|
|
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
|
|
hc2.recordStatus('svc1', makeDown()); // 3 — flip
|
|
expect(spy).toHaveBeenCalledTimes(2);
|
|
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
|
|
});
|
|
|
|
test.each(['not-a-number', '0', '-2', '1.5'])('malformed DOWN_THRESHOLD %s falls back to 2', value => {
|
|
process.env.HEALTH_DOWN_THRESHOLD = value;
|
|
jest.resetModules();
|
|
const hc2 = require('../src/monitoring/health-checker');
|
|
hc2.displayedStatus = new Map();
|
|
hc2.consecutiveSinceChange = new Map();
|
|
hc2.currentStatus = new Map();
|
|
hc2.history = {};
|
|
hc2.removeAllListeners('status-check');
|
|
const spy = jest.fn();
|
|
hc2.on('status-check', spy);
|
|
hc2.recordStatus('svc1', makeUp());
|
|
hc2.recordStatus('svc1', makeDown());
|
|
expect(spy).toHaveBeenCalledTimes(1);
|
|
hc2.recordStatus('svc1', makeDown());
|
|
expect(spy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
test('UP_THRESHOLD env var greater than 1 is honored', () => {
|
|
process.env.HEALTH_UP_THRESHOLD = '2';
|
|
jest.resetModules();
|
|
const hc2 = require('../src/monitoring/health-checker');
|
|
hc2.displayedStatus = new Map();
|
|
hc2.consecutiveSinceChange = new Map();
|
|
hc2.currentStatus = new Map();
|
|
hc2.history = {};
|
|
hc2.removeAllListeners('status-check');
|
|
const spy = jest.fn();
|
|
hc2.on('status-check', spy);
|
|
hc2.recordStatus('svc1', makeDown());
|
|
hc2.recordStatus('svc1', makeUp());
|
|
expect(spy).toHaveBeenCalledTimes(1);
|
|
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
|
|
hc2.recordStatus('svc1', makeUp());
|
|
expect(spy).toHaveBeenCalledTimes(2);
|
|
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
|
|
});
|
|
|
|
test.each(['not-a-number', '0', '-2', '1.5'])('malformed UP_THRESHOLD %s falls back to 1', value => {
|
|
process.env.HEALTH_UP_THRESHOLD = value;
|
|
jest.resetModules();
|
|
const hc2 = require('../src/monitoring/health-checker');
|
|
hc2.displayedStatus = new Map();
|
|
hc2.consecutiveSinceChange = new Map();
|
|
hc2.currentStatus = new Map();
|
|
hc2.history = {};
|
|
hc2.removeAllListeners('status-check');
|
|
const spy = jest.fn();
|
|
hc2.on('status-check', spy);
|
|
hc2.recordStatus('svc1', makeDown());
|
|
hc2.recordStatus('svc1', makeUp());
|
|
expect(spy).toHaveBeenCalledTimes(2);
|
|
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
|
|
});
|
|
|
|
test('removeService clears hysteresis state before the same ID is re-added', () => {
|
|
hc.config.services.svc1 = { name: 'Service 1' };
|
|
hc.recordStatus('svc1', makeUp());
|
|
hc.recordStatus('svc1', makeDown());
|
|
expect(hc.displayedStatus.has('svc1')).toBe(true);
|
|
expect(hc.consecutiveSinceChange.get('svc1')).toBe(1);
|
|
hc.consecutiveFailures.set('svc1', 3);
|
|
const timer = setTimeout(() => {}, 60_000);
|
|
hc.serviceTimers.set('svc1', timer);
|
|
|
|
hc.saveConfig = jest.fn();
|
|
hc.removeService('svc1');
|
|
|
|
expect(hc.displayedStatus.has('svc1')).toBe(false);
|
|
expect(hc.consecutiveSinceChange.has('svc1')).toBe(false);
|
|
expect(hc.currentStatus.has('svc1')).toBe(false);
|
|
expect(hc.consecutiveFailures.has('svc1')).toBe(false);
|
|
expect(hc.serviceTimers.has('svc1')).toBe(false);
|
|
|
|
hc.config.services.svc1 = { name: 'Service 1 re-added' };
|
|
const emitSpyAfterReAdd = jest.fn();
|
|
hc.on('status-check', emitSpyAfterReAdd);
|
|
hc.recordStatus('svc1', makeDown());
|
|
|
|
expect(emitSpyAfterReAdd).toHaveBeenCalledTimes(1);
|
|
expect(hc.displayedStatus.get('svc1').status).toBe('down');
|
|
expect(hc.consecutiveSinceChange.has('svc1')).toBe(false);
|
|
});
|
|
});
|