Compare commits

..
Author SHA1 Message Date
Hermes 88f1d4a414 [glm-grade=A] fix(monitoring): DC-090 outage incidents follow displayed hysteresis status
checkForIncidents compared raw probe transitions while the dashboard badge
(DC-086) follows post-hysteresis displayed status. A single raw down blip
between two ups opened AND resolved a critical outage incident; a suppressed
up blip during a real outage resolved it early. Incidents now open/resolve
on displayed-vs-displayed transitions; previousDisplayed=null keeps legacy
raw semantics for direct callers. 6 new parity tests + legacy checkService
test moved to a 4-probe chain. Suite 2616/2616 (110).

Verdict: urn:ump:yc5rdlnmnmhch5audc5fifgbt6d7moi2stqfs5vidsbh6x6zkvgq
2026-08-22 16:52:53 -07:00
Hermes dd1110ef52 Merge dc/DC-089-invite-log-pii-masking: mask invite/user emails in server logs [glm-grade=B]
CI / Test & Lint (push) Canceled after 0s
CI / Security audit (push) Canceled after 0s
2026-08-22 16:23:06 -07:00
Hermes 6732a1e1df [glm-grade=B] fix(auth): DC-089 mask invite/user emails in server logs
Two log sites in routes/auth/admin.js wrote raw email PII to the server
log: the SMTP-unconfigured 'auth-invite-send' warn and the 'invite
accepted, user created' info. Both now route through
AuthProvider.maskEmail() with a '[unmaskable-email]' sentinel fallback
(never the raw address). Two regression tests assert the raw address is
absent from log meta and the masked form present. Response contract
unchanged (full email still returned to the authenticated admin).

Judge: GLM-5.3 cold read (deleg_f0896de3), grade B / ship / zero
blockers; polish notes folded in. Verdict URN:
urn:ump:jd2htpwq76ni6bapj3vxjpvypfdnoc4argqbzpcerutmh7u5khea
Full suite: 2610/2610 (109 suites).
2026-08-22 16:22:53 -07:00
Hermes ea96abe95a Merge dc/DC-088-remove-service-tombstones: removeService generation tombstones + incident closure [glm-grade=B]
CI / Test & Lint (push) Canceled after 0s
CI / Security audit (push) Canceled after 0s
2026-08-22 15:55:29 -07:00
Hermes 3ccf66754a [glm-grade=B] fix(monitoring): DC-088 removeService generation tombstones + incident closure
- serviceGenerations no longer leaks entries: removeService deletes the live
  entry and records a TTL'd (10min) tombstone swept by cleanupHistory
- monotonic instance-wide generationSeq prevents generation reuse across
  remove->re-add cycles (ABA) and supersedes tombstones on re-configure
- _isStaleCapture(): presence-aware stale check — live entry must match
  exactly; no entry is stale only under a higher-generation tombstone
  (preserves correct behavior for disk-loaded never-configured services)
- catch path increments consecutiveFailures only after the stale check, so
  a late-rejected probe cannot resurrect state for a removed service
- open incidents for a removed service close via the standard resolve path
  (resolvedBy=service-removed, WS/SSE incident-resolved broadcast)
- 6 regression tests; full suite 2608/2608 green

Judge: GLM-5.3 cold read (Codex stand-in), verdict B/ship, zero blockers
2026-08-22 15:55:26 -07:00
Hermes c71b794ccc Merge dc/DC-087-test-mirrors-fetcht: hermetic caddy-admin test mirrors + raw-fetch guard [glm-grade=B]
CI / Test & Lint (push) Canceled after 0s
CI / Security audit (push) Canceled after 0s
2026-08-22 15:14:35 -07:00
5 changed files with 450 additions and 16 deletions
+46 -1
View File
@@ -237,4 +237,49 @@ describe('DC-085: link-first admin invites', () => {
.send({ email: 'a@x.com', ttlHours: 1 }); .send({ email: 'a@x.com', ttlHours: 1 });
expect(res.body.shareText).toContain('expires in 1h'); expect(res.body.shareText).toContain('expires in 1h');
}); });
});
test('DC-089: SMTP-unconfigured warn log masks the invite email (no raw PII)', async () => {
mockEmailSender.isConfigured.mockReturnValueOnce(false);
const res = await request(app)
.post('/api/v1/auth/admin/invites')
.send({ email: 'friend@example.com', role: 'operator', sendEmail: true });
expect(res.status).toBe(200);
expect(res.body.deliveredVia).toBe('failed');
const warn = logCalls.find(c =>
c.level === 'warn' && c.topic === 'auth-invite-send'
);
expect(warn).toBeDefined();
// The raw address must not appear; the masked form must.
expect(JSON.stringify(warn.meta)).not.toContain('friend@example.com');
expect(warn.meta.email).toBe('fr****@example.com');
});
test('DC-089: invite-accepted info log masks the created user email (no raw PII)', async () => {
// Pre-authorize the email (POST /admin/users) so userStore.login doesn't
// reject with not_authorized — bootstrap already happened in beforeEach.
const preauth = await request(app)
.post('/api/v1/auth/admin/users')
.send({ email: 'newfriend@example.com' });
expect(preauth.status).toBe(200);
const issue = await request(app)
.post('/api/v1/auth/admin/invites')
.send({ email: 'newfriend@example.com', role: 'viewer' });
expect(issue.status).toBe(200);
const token = issue.body.acceptUrl.match(/invites\/([^/]+)\/accept/)[1];
const res = await request(app)
.post(`/api/v1/auth/invites/${token}/accept`)
.send({});
expect(res.status).toBe(200);
const info = logCalls.find(c =>
c.level === 'info' && c.msg === 'invite accepted, user created'
);
expect(info).toBeDefined();
expect(JSON.stringify(info.meta)).not.toContain('newfriend@example.com');
expect(info.meta.email).toBe('ne****@example.com');
});
});
@@ -0,0 +1,186 @@
/**
* Tests for DC-090: outage incidents follow the DISPLAYED (post-hysteresis)
* status — the same signal that flips the dashboard badge.
*
* - A single raw "down" blip that hysteresis suppresses opens NO outage
* incident (the DC-089-noted raw-transition bug).
* - A suppressed blip does not resolve a real open outage (UP_THRESHOLD=2).
* - DOWN_THRESHOLD consecutive downs open exactly ONE outage incident.
* - The incident payload carries the displayed snapshot, not the raw probe.
* - Direct callers without hysteresis state keep legacy raw semantics.
*
* The probe() helper replicates checkService's exact call order: capture the
* pre-probe raw + displayed state, recordStatus (updates both maps), then
* checkForIncidents with both previous states.
*/
'use strict';
const path = require('path');
const fs = require('fs');
const os = require('os');
// Use an isolated data dir so test history doesn't pollute the real one.
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dashcaddy-incpar-'));
process.env.HEALTH_DATA_DIR = tmpDir;
process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
// Module exports a singleton instance, not a class. Reset per-test state by
// replacing the relevant maps on the singleton in beforeEach.
const healthCheckerSingleton = require('../src/monitoring/health-checker');
const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD;
const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD;
function restoreEnv(name, value) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
function makeUp(serviceId = 'svc1') {
return {
serviceId,
timestamp: new Date().toISOString(),
status: 'up',
responseTime: 50,
statusCode: 200,
message: 'Service is healthy',
details: { headers: {}, bodyLength: 12 }
};
}
function makeDown(serviceId = 'svc1') {
return {
serviceId,
timestamp: new Date().toISOString(),
status: 'down',
responseTime: 50,
statusCode: 500,
message: 'fail',
details: { headers: {}, bodyLength: 0 }
};
}
describe('DC-090: outage incidents follow the displayed (hysteresis) status', () => {
let hc;
let incidentCreatedSpy;
let incidentResolvedSpy;
beforeEach(() => {
healthCheckerSingleton.displayedStatus = new Map();
healthCheckerSingleton.consecutiveSinceChange = new Map();
healthCheckerSingleton.currentStatus = new Map();
healthCheckerSingleton.history = {};
healthCheckerSingleton.incidents = [];
healthCheckerSingleton.removeAllListeners('incident-created');
healthCheckerSingleton.removeAllListeners('incident-resolved');
incidentCreatedSpy = jest.fn();
incidentResolvedSpy = jest.fn();
healthCheckerSingleton.on('incident-created', incidentCreatedSpy);
healthCheckerSingleton.on('incident-resolved', incidentResolvedSpy);
hc = healthCheckerSingleton;
});
afterEach(() => {
restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold);
restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold);
});
afterAll(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
// Replicates checkService's record+incident sequence for one raw probe.
function probe(status, config = {}) {
const previousStatus = hc.currentStatus.get(status.serviceId);
const previousDisplayed = hc.displayedStatus.get(status.serviceId) || null;
hc.recordStatus(status.serviceId, status);
hc.checkForIncidents(status.serviceId, status, config, previousStatus, previousDisplayed);
}
test('a single down blip between two ups opens NO outage incident', () => {
probe(makeUp()); // baseline: displayed up
probe(makeDown()); // blip — hysteresis keeps displayed up
probe(makeUp()); // recovered
expect(hc.incidents).toHaveLength(0);
expect(incidentCreatedSpy).not.toHaveBeenCalled();
});
test('DOWN_THRESHOLD consecutive downs open exactly one outage incident (critical)', () => {
probe(makeUp());
probe(makeDown()); // counter=1, displayed still up
probe(makeDown()); // counter=2 → displayed flips down → incident
expect(hc.incidents).toHaveLength(1);
const incident = hc.incidents[0];
expect(incident.type).toBe('outage');
expect(incident.severity).toBe('critical');
expect(incident.status).toBe('open');
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
probe(makeDown()); // still down — no new transition, no second incident
expect(hc.incidents).toHaveLength(1);
expect(incident.occurrences).toBe(1); // occurrences count displayed flips, not raw probes
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
});
test('the outage incident payload carries the displayed snapshot, not the raw blip', () => {
probe(makeUp());
const blip = makeDown();
blip.statusCode = 599;
probe(blip); // suppressed blip — must not appear in any incident
probe(makeDown()); // flip
expect(hc.incidents).toHaveLength(1);
// The incident's details snapshot is the probe that FLIPPED the displayed
// state (the second down), not the earlier suppressed blip.
expect(hc.incidents[0].details.statusCode).not.toBe(599);
});
test('a suppressed up blip does not resolve a real open outage (UP_THRESHOLD=2)', () => {
process.env.HEALTH_UP_THRESHOLD = '2';
jest.resetModules();
const hc2 = require('../src/monitoring/health-checker');
hc2.displayedStatus = new Map();
hc2.consecutiveSinceChange = new Map();
hc2.currentStatus = new Map();
hc2.history = {};
hc2.incidents = [];
hc2.removeAllListeners('incident-created');
hc2.removeAllListeners('incident-resolved');
const p2 = (status) => {
const prevRaw = hc2.currentStatus.get(status.serviceId);
const prevDisp = hc2.displayedStatus.get(status.serviceId) || null;
hc2.recordStatus(status.serviceId, status);
hc2.checkForIncidents(status.serviceId, status, {}, prevRaw, prevDisp);
};
p2(makeUp());
p2(makeDown());
p2(makeDown()); // displayed down → outage opens
expect(hc2.incidents).toHaveLength(1);
expect(hc2.incidents[0].status).toBe('open');
p2(makeUp()); // counter=1 < UP_THRESHOLD=2 → displayed still down
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
expect(hc2.incidents[0].status).toBe('open'); // NOT resolved by the blip
p2(makeUp()); // counter=2 → displayed up → incident resolves
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
expect(hc2.incidents[0].status).toBe('resolved');
});
test('legacy direct callers (no displayed state) keep raw transition semantics', () => {
hc.currentStatus.set('svc1', { status: 'up' });
const status = { status: 'down', timestamp: new Date().toISOString(), responseTime: 100 };
hc.checkForIncidents('svc1', status, {}); // 4-arg call, no previousDisplayed
expect(hc.incidents).toHaveLength(1);
expect(hc.incidents[0].type).toBe('outage');
});
test('slow-response detection still fires per-probe regardless of hysteresis', () => {
const slowUp = makeUp();
slowUp.responseTime = 6000;
probe(slowUp, { slowResponseThreshold: 5000 });
expect(hc.incidents.some(i => i.type === 'slow-response')).toBe(true);
});
});
+115 -1
View File
@@ -204,15 +204,22 @@ describe('HealthChecker', () => {
}); });
it('opens and resolves an outage incident across real checkService transitions', async () => { it('opens and resolves an outage incident across real checkService transitions', async () => {
// DC-090: incidents follow the DISPLAYED (post-hysteresis) status.
// DOWN_THRESHOLD defaults to 2, so it takes two consecutive failed
// probes to flip displayed down and open the outage; one up probe
// (UP_THRESHOLD=1) resolves it.
healthChecker._doRequest = jest.fn() healthChecker._doRequest = jest.fn()
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} }) .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} }) .mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} }); .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });
const config = { url: 'http://test.local' }; const config = { url: 'http://test.local' };
await healthChecker.checkService('svc1', config); await healthChecker.checkService('svc1', config);
await healthChecker.checkService('svc1', config); await healthChecker.checkService('svc1', config);
expect(healthChecker.incidents).toHaveLength(0); // one down alone: suppressed blip
await healthChecker.checkService('svc1', config); // second down flips displayed → open
expect(healthChecker.incidents).toHaveLength(1); expect(healthChecker.incidents).toHaveLength(1);
expect(healthChecker.incidents[0]).toMatchObject({ expect(healthChecker.incidents[0]).toMatchObject({
serviceId: 'svc1', serviceId: 'svc1',
@@ -220,7 +227,7 @@ describe('HealthChecker', () => {
status: 'open' status: 'open'
}); });
await healthChecker.checkService('svc1', config); await healthChecker.checkService('svc1', config); // up resolves
expect(healthChecker.incidents[0].status).toBe('resolved'); expect(healthChecker.incidents[0].status).toBe('resolved');
expect(healthChecker.incidents[0].resolvedAt).toBeDefined(); expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
}); });
@@ -591,6 +598,113 @@ describe('HealthChecker', () => {
}); });
}); });
describe('DC-088: removeService generation tombstones + incident closure', () => {
it('does not leak a serviceGenerations entry and records a tombstone', () => {
healthChecker.configureService('svc1', { url: 'http://test.local' });
expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);
healthChecker.removeService('svc1');
expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
const tomb = healthChecker.removedGenerations.get('svc1');
expect(tomb).toBeDefined();
expect(tomb.generation).toBeGreaterThan(0);
expect(tomb.removedAt).toBeGreaterThan(0);
});
it('re-added service gets a strictly higher generation (no ABA)', () => {
healthChecker.configureService('svc1', { url: 'http://test.local' });
const gen1 = healthChecker.serviceGenerations.get('svc1');
healthChecker.removeService('svc1');
healthChecker.configureService('svc1', { url: 'http://test.local/v2' });
const gen2 = healthChecker.serviceGenerations.get('svc1');
expect(gen2).toBeGreaterThan(gen1);
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
});
it('closes open incidents for the removed service as resolved', () => {
healthChecker.saveConfig = jest.fn();
healthChecker.incidents.push({
id: 'incident-test-1',
serviceId: 'svc1',
type: 'outage',
status: 'open',
createdAt: new Date(Date.now() - 60_000).toISOString()
});
healthChecker.incidents.push({
id: 'incident-other',
serviceId: 'svc2',
type: 'outage',
status: 'open',
createdAt: new Date(Date.now() - 60_000).toISOString()
});
const resolvedSpy = jest.fn();
healthChecker.on('incident-resolved', resolvedSpy);
healthChecker.removeService('svc1');
const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
expect(closed.status).toBe('resolved');
expect(closed.resolvedBy).toBe('service-removed');
expect(closed.resolvedAt).toBeDefined();
expect(closed.duration).toBeGreaterThan(0);
expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
expect(resolvedSpy).toHaveBeenCalledTimes(1);
});
it('in-flight probe captured before removal is discarded via tombstone', async () => {
let resolveProbe;
healthChecker.config.services.svc1 = { url: 'http://test.local' };
healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
resolveProbe = resolve;
}));
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
healthChecker.saveConfig = jest.fn();
healthChecker.removeService('svc1');
resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
await pending;
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
});
it('a rejected in-flight probe after removal does not re-create failure state', async () => {
let rejectProbe;
healthChecker.config.services.svc1 = { url: 'http://test.local' };
healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
rejectProbe = reject;
}));
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
healthChecker.saveConfig = jest.fn();
healthChecker.removeService('svc1');
rejectProbe(new Error('late failure'));
await pending;
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
});
it('sweeps expired tombstones in cleanupHistory', () => {
healthChecker.removedGenerations.set('svc1', {
generation: 1,
removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
});
healthChecker.removedGenerations.set('svc2', {
generation: 2,
removedAt: Date.now() // fresh
});
healthChecker.cleanupHistory();
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
});
});
describe('cleanupHistory', () => { describe('cleanupHistory', () => {
it('removes entries older than retention period', () => { it('removes entries older than retention period', () => {
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
+3 -2
View File
@@ -29,6 +29,7 @@ const platformPaths = require('../../platform-paths');
const { createUserStore } = require('../../src/security/user-store'); const { createUserStore } = require('../../src/security/user-store');
const { createInviteStore } = require('../../src/security/invite-store'); const { createInviteStore } = require('../../src/security/invite-store');
const emailSender = require('../../src/auth/providers/email-sender'); const emailSender = require('../../src/auth/providers/email-sender');
const AuthProvider = require('../../src/auth/providers/base');
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors'); const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
const { ok, successMessage } = require('../../src/utils/responses'); const { ok, successMessage } = require('../../src/utils/responses');
@@ -254,7 +255,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
// server log on every unconfigured-install invite. // server log on every unconfigured-install invite.
log.warn && log.warn('auth-invite-send', log.warn && log.warn('auth-invite-send',
'invite send skipped: SMTP not configured (operator opted in)', 'invite send skipped: SMTP not configured (operator opted in)',
{ inviteId: issued.id, email: issued.email }); { inviteId: issued.id, email: AuthProvider.maskEmail(issued.email) || '[unmaskable-email]' });
deliveredVia = 'failed'; deliveredVia = 'failed';
} }
} catch (sendErr) { } catch (sendErr) {
@@ -369,7 +370,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
log.info && log.info('auth', 'invite accepted, user created', { log.info && log.info('auth', 'invite accepted, user created', {
userId: userResult.user.id, userId: userResult.user.id,
email: userResult.user.email, email: AuthProvider.maskEmail(userResult.user.email) || '[unmaskable-email]',
role: userResult.user.role, role: userResult.user.role,
inviteId: invite.id, inviteId: invite.id,
}); });
+100 -12
View File
@@ -32,6 +32,10 @@ const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10); const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
// DC-088: how long a removal tombstone outlives the removal itself. Only needs
// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
// swept by cleanupHistory so removed services cannot accumulate map entries.
const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
// DC-086: hysteresis thresholds for badge display. // DC-086: hysteresis thresholds for badge display.
// The raw probe result can flap on a single transient blip (Caddy reload, // The raw probe result can flap on a single transient blip (Caddy reload,
@@ -72,6 +76,14 @@ class HealthChecker extends EventEmitter {
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
// Invalidate probe completions that race with removal/reconfiguration. // Invalidate probe completions that race with removal/reconfiguration.
this.serviceGenerations = new Map(); // serviceId -> configuration generation this.serviceGenerations = new Map(); // serviceId -> configuration generation
// DC-088: monotonically increasing sequence so generation numbers can never
// repeat across remove -> re-add cycles (prevents ABA on the stale check).
this.generationSeq = 0;
// DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
// serviceGenerations means the service is (re)configured; a tombstone with a
// HIGHER generation than the captured one marks the capture as stale. Entry
// is deleted when the service is removed, so the live map cannot leak.
this.removedGenerations = new Map();
} }
/** /**
@@ -139,6 +151,21 @@ class HealthChecker extends EventEmitter {
this.cleanupHistory(); this.cleanupHistory();
} }
/**
* DC-088: true when a probe's captured generation no longer matches the
* service's current configuration state. A live serviceGenerations entry
* must match exactly. With no live entry the service was never configured
* in this process (disk-loaded / direct callers) stale only if a removal
* tombstone with a HIGHER generation exists.
*/
_isStaleCapture(serviceId, generation) {
if (this.serviceGenerations.has(serviceId)) {
return this.serviceGenerations.get(serviceId) !== generation;
}
const tomb = this.removedGenerations.get(serviceId);
return Boolean(tomb && tomb.generation > generation);
}
/** /**
* Check a single service * Check a single service
*/ */
@@ -160,7 +187,7 @@ class HealthChecker extends EventEmitter {
details: result.details details: result.details
}; };
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) { if (this._isStaleCapture(serviceId, generation)) {
return status; return status;
} }
@@ -172,16 +199,14 @@ class HealthChecker extends EventEmitter {
} }
const previousStatus = this.currentStatus.get(serviceId); const previousStatus = this.currentStatus.get(serviceId);
const previousDisplayed = this.displayedStatus.get(serviceId);
this.recordStatus(serviceId, status); this.recordStatus(serviceId, status);
this.checkForIncidents(serviceId, status, config, previousStatus); this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
return status; return status;
} catch (error) { } catch (error) {
const responseTime = Date.now() - startTime; const responseTime = Date.now() - startTime;
// Increment failure count for backoff
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
const status = { const status = {
serviceId, serviceId,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
@@ -190,13 +215,18 @@ class HealthChecker extends EventEmitter {
error: error.message error: error.message
}; };
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) { if (this._isStaleCapture(serviceId, generation)) {
return status; return status;
} }
// Increment failure count for backoff — only after the result is known
// to be non-stale, so a removed service cannot re-create map entries.
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
const previousStatus = this.currentStatus.get(serviceId); const previousStatus = this.currentStatus.get(serviceId);
const previousDisplayed = this.displayedStatus.get(serviceId);
this.recordStatus(serviceId, status); this.recordStatus(serviceId, status);
this.checkForIncidents(serviceId, status, config, previousStatus); this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
return status; return status;
} }
@@ -425,10 +455,27 @@ class HealthChecker extends EventEmitter {
/** /**
* Check for incidents (downtime, slow response, etc.) * Check for incidents (downtime, slow response, etc.)
*/ */
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId)) { checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
// Check for status change (up -> down or down -> up) // DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
if (previous && previous.status !== status.status) { // the same signal that flips the dashboard badge. A single raw "down"
// blip that hysteresis suppresses must not open a critical outage
// incident (and a suppressed blip must not resolve a real one). When the
// caller supplies the pre-probe displayed state (checkService always
// does), transitions are evaluated displayed-vs-displayed using the
// post-recordStatus state in this.displayedStatus. Direct callers with
// no hysteresis state (previousDisplayed === null) keep the legacy
// raw-probe transition semantics.
if (previousDisplayed) {
const displayed = this.displayedStatus.get(serviceId);
if (displayed && displayed.status !== previousDisplayed.status) {
if (displayed.status === 'down') {
this.createIncident(serviceId, 'outage', 'Service is down', displayed);
} else if (displayed.status === 'up') {
this.resolveIncident(serviceId, 'outage', displayed);
}
}
} else if (previous && previous.status !== status.status) {
if (status.status === 'down') { if (status.status === 'down') {
this.createIncident(serviceId, 'outage', 'Service is down', status); this.createIncident(serviceId, 'outage', 'Service is down', status);
} else if (status.status === 'up') { } else if (status.status === 'up') {
@@ -660,7 +707,13 @@ class HealthChecker extends EventEmitter {
this.config.services = {}; this.config.services = {};
} }
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1); // DC-088: monotonic instance-wide sequence — a re-added service can never
// recycle a previous generation number, and any older in-flight capture is
// invalidated by definition.
this.generationSeq += 1;
this.serviceGenerations.set(serviceId, this.generationSeq);
// Re-configuration supersedes any prior removal tombstone.
this.removedGenerations.delete(serviceId);
this.config.services[serviceId] = { this.config.services[serviceId] = {
enabled: config.enabled !== false, enabled: config.enabled !== false,
name: config.name || serviceId, name: config.name || serviceId,
@@ -683,12 +736,35 @@ class HealthChecker extends EventEmitter {
* Remove service configuration * Remove service configuration
*/ */
removeService(serviceId) { removeService(serviceId) {
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1); // DC-088: tombstone the captured generation instead of leaking an entry.
// The live map entry is deleted; an in-flight probe captured BEFORE this
// point sees no live entry but a higher tombstone generation, so it is
// discarded. configureService clears the tombstone on re-add.
this.generationSeq += 1;
this.serviceGenerations.delete(serviceId);
this.removedGenerations.set(serviceId, {
generation: this.generationSeq,
removedAt: Date.now()
});
if (this.config.services) { if (this.config.services) {
delete this.config.services[serviceId]; delete this.config.services[serviceId];
this.saveConfig(); this.saveConfig();
} }
// DC-088: open incidents for a removed service must not linger forever.
// Close them through the same resolve path a recovery would, annotated so
// history shows why (dashboard renders resolved incidents green + duration).
for (const incident of this.incidents) {
if (incident.serviceId === serviceId && incident.status === 'open') {
incident.status = 'resolved';
incident.resolvedAt = new Date().toISOString();
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
incident.resolvedBy = 'service-removed';
this.emit('incident-resolved', incident);
this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
}
}
this.currentStatus.delete(serviceId); this.currentStatus.delete(serviceId);
this.displayedStatus.delete(serviceId); this.displayedStatus.delete(serviceId);
this.consecutiveSinceChange.delete(serviceId); this.consecutiveSinceChange.delete(serviceId);
@@ -714,6 +790,18 @@ class HealthChecker extends EventEmitter {
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE); this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
} }
} }
// DC-088: sweep expired removal tombstones. After the TTL no probe that
// captured a pre-removal generation can still be in flight (timeout is
// bounded by performHealthCheck), so the tombstone has done its job.
if (this.removedGenerations.size > 0) {
const now = Date.now();
for (const [serviceId, tomb] of this.removedGenerations) {
if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
this.removedGenerations.delete(serviceId);
}
}
}
} }
/** /**