Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
88f1d4a414 | ||
|
|
dd1110ef52 | ||
|
|
6732a1e1df | ||
|
|
ea96abe95a | ||
|
|
3ccf66754a | ||
|
|
c71b794ccc |
@@ -237,4 +237,49 @@ describe('DC-085: link-first admin invites', () => {
|
|||||||
.send({ email: 'a@x.com', ttlHours: 1 });
|
.send({ email: 'a@x.com', ttlHours: 1 });
|
||||||
expect(res.body.shareText).toContain('expires in 1h');
|
expect(res.body.shareText).toContain('expires in 1h');
|
||||||
});
|
});
|
||||||
});
|
|
||||||
|
test('DC-089: SMTP-unconfigured warn log masks the invite email (no raw PII)', async () => {
|
||||||
|
mockEmailSender.isConfigured.mockReturnValueOnce(false);
|
||||||
|
|
||||||
|
const res = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/invites')
|
||||||
|
.send({ email: 'friend@example.com', role: 'operator', sendEmail: true });
|
||||||
|
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.body.deliveredVia).toBe('failed');
|
||||||
|
const warn = logCalls.find(c =>
|
||||||
|
c.level === 'warn' && c.topic === 'auth-invite-send'
|
||||||
|
);
|
||||||
|
expect(warn).toBeDefined();
|
||||||
|
// The raw address must not appear; the masked form must.
|
||||||
|
expect(JSON.stringify(warn.meta)).not.toContain('friend@example.com');
|
||||||
|
expect(warn.meta.email).toBe('fr****@example.com');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('DC-089: invite-accepted info log masks the created user email (no raw PII)', async () => {
|
||||||
|
// Pre-authorize the email (POST /admin/users) so userStore.login doesn't
|
||||||
|
// reject with not_authorized — bootstrap already happened in beforeEach.
|
||||||
|
const preauth = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/users')
|
||||||
|
.send({ email: 'newfriend@example.com' });
|
||||||
|
expect(preauth.status).toBe(200);
|
||||||
|
|
||||||
|
const issue = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/invites')
|
||||||
|
.send({ email: 'newfriend@example.com', role: 'viewer' });
|
||||||
|
expect(issue.status).toBe(200);
|
||||||
|
const token = issue.body.acceptUrl.match(/invites\/([^/]+)\/accept/)[1];
|
||||||
|
|
||||||
|
const res = await request(app)
|
||||||
|
.post(`/api/v1/auth/invites/${token}/accept`)
|
||||||
|
.send({});
|
||||||
|
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
const info = logCalls.find(c =>
|
||||||
|
c.level === 'info' && c.msg === 'invite accepted, user created'
|
||||||
|
);
|
||||||
|
expect(info).toBeDefined();
|
||||||
|
expect(JSON.stringify(info.meta)).not.toContain('newfriend@example.com');
|
||||||
|
expect(info.meta.email).toBe('ne****@example.com');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
/**
|
||||||
|
* Tests for DC-090: outage incidents follow the DISPLAYED (post-hysteresis)
|
||||||
|
* status — the same signal that flips the dashboard badge.
|
||||||
|
*
|
||||||
|
* - A single raw "down" blip that hysteresis suppresses opens NO outage
|
||||||
|
* incident (the DC-089-noted raw-transition bug).
|
||||||
|
* - A suppressed blip does not resolve a real open outage (UP_THRESHOLD=2).
|
||||||
|
* - DOWN_THRESHOLD consecutive downs open exactly ONE outage incident.
|
||||||
|
* - The incident payload carries the displayed snapshot, not the raw probe.
|
||||||
|
* - Direct callers without hysteresis state keep legacy raw semantics.
|
||||||
|
*
|
||||||
|
* The probe() helper replicates checkService's exact call order: capture the
|
||||||
|
* pre-probe raw + displayed state, recordStatus (updates both maps), then
|
||||||
|
* checkForIncidents with both previous states.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
|
||||||
|
// Use an isolated data dir so test history doesn't pollute the real one.
|
||||||
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dashcaddy-incpar-'));
|
||||||
|
process.env.HEALTH_DATA_DIR = tmpDir;
|
||||||
|
process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
|
||||||
|
process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
|
||||||
|
|
||||||
|
// Module exports a singleton instance, not a class. Reset per-test state by
|
||||||
|
// replacing the relevant maps on the singleton in beforeEach.
|
||||||
|
const healthCheckerSingleton = require('../src/monitoring/health-checker');
|
||||||
|
const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD;
|
||||||
|
const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD;
|
||||||
|
|
||||||
|
function restoreEnv(name, value) {
|
||||||
|
if (value === undefined) delete process.env[name];
|
||||||
|
else process.env[name] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeUp(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'up',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 200,
|
||||||
|
message: 'Service is healthy',
|
||||||
|
details: { headers: {}, bodyLength: 12 }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDown(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'down',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 500,
|
||||||
|
message: 'fail',
|
||||||
|
details: { headers: {}, bodyLength: 0 }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DC-090: outage incidents follow the displayed (hysteresis) status', () => {
|
||||||
|
let hc;
|
||||||
|
let incidentCreatedSpy;
|
||||||
|
let incidentResolvedSpy;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
healthCheckerSingleton.displayedStatus = new Map();
|
||||||
|
healthCheckerSingleton.consecutiveSinceChange = new Map();
|
||||||
|
healthCheckerSingleton.currentStatus = new Map();
|
||||||
|
healthCheckerSingleton.history = {};
|
||||||
|
healthCheckerSingleton.incidents = [];
|
||||||
|
healthCheckerSingleton.removeAllListeners('incident-created');
|
||||||
|
healthCheckerSingleton.removeAllListeners('incident-resolved');
|
||||||
|
incidentCreatedSpy = jest.fn();
|
||||||
|
incidentResolvedSpy = jest.fn();
|
||||||
|
healthCheckerSingleton.on('incident-created', incidentCreatedSpy);
|
||||||
|
healthCheckerSingleton.on('incident-resolved', incidentResolvedSpy);
|
||||||
|
hc = healthCheckerSingleton;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold);
|
||||||
|
restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Replicates checkService's record+incident sequence for one raw probe.
|
||||||
|
function probe(status, config = {}) {
|
||||||
|
const previousStatus = hc.currentStatus.get(status.serviceId);
|
||||||
|
const previousDisplayed = hc.displayedStatus.get(status.serviceId) || null;
|
||||||
|
hc.recordStatus(status.serviceId, status);
|
||||||
|
hc.checkForIncidents(status.serviceId, status, config, previousStatus, previousDisplayed);
|
||||||
|
}
|
||||||
|
|
||||||
|
test('a single down blip between two ups opens NO outage incident', () => {
|
||||||
|
probe(makeUp()); // baseline: displayed up
|
||||||
|
probe(makeDown()); // blip — hysteresis keeps displayed up
|
||||||
|
probe(makeUp()); // recovered
|
||||||
|
expect(hc.incidents).toHaveLength(0);
|
||||||
|
expect(incidentCreatedSpy).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('DOWN_THRESHOLD consecutive downs open exactly one outage incident (critical)', () => {
|
||||||
|
probe(makeUp());
|
||||||
|
probe(makeDown()); // counter=1, displayed still up
|
||||||
|
probe(makeDown()); // counter=2 → displayed flips down → incident
|
||||||
|
expect(hc.incidents).toHaveLength(1);
|
||||||
|
const incident = hc.incidents[0];
|
||||||
|
expect(incident.type).toBe('outage');
|
||||||
|
expect(incident.severity).toBe('critical');
|
||||||
|
expect(incident.status).toBe('open');
|
||||||
|
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
probe(makeDown()); // still down — no new transition, no second incident
|
||||||
|
expect(hc.incidents).toHaveLength(1);
|
||||||
|
expect(incident.occurrences).toBe(1); // occurrences count displayed flips, not raw probes
|
||||||
|
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the outage incident payload carries the displayed snapshot, not the raw blip', () => {
|
||||||
|
probe(makeUp());
|
||||||
|
const blip = makeDown();
|
||||||
|
blip.statusCode = 599;
|
||||||
|
probe(blip); // suppressed blip — must not appear in any incident
|
||||||
|
probe(makeDown()); // flip
|
||||||
|
expect(hc.incidents).toHaveLength(1);
|
||||||
|
// The incident's details snapshot is the probe that FLIPPED the displayed
|
||||||
|
// state (the second down), not the earlier suppressed blip.
|
||||||
|
expect(hc.incidents[0].details.statusCode).not.toBe(599);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a suppressed up blip does not resolve a real open outage (UP_THRESHOLD=2)', () => {
|
||||||
|
process.env.HEALTH_UP_THRESHOLD = '2';
|
||||||
|
jest.resetModules();
|
||||||
|
const hc2 = require('../src/monitoring/health-checker');
|
||||||
|
hc2.displayedStatus = new Map();
|
||||||
|
hc2.consecutiveSinceChange = new Map();
|
||||||
|
hc2.currentStatus = new Map();
|
||||||
|
hc2.history = {};
|
||||||
|
hc2.incidents = [];
|
||||||
|
hc2.removeAllListeners('incident-created');
|
||||||
|
hc2.removeAllListeners('incident-resolved');
|
||||||
|
|
||||||
|
const p2 = (status) => {
|
||||||
|
const prevRaw = hc2.currentStatus.get(status.serviceId);
|
||||||
|
const prevDisp = hc2.displayedStatus.get(status.serviceId) || null;
|
||||||
|
hc2.recordStatus(status.serviceId, status);
|
||||||
|
hc2.checkForIncidents(status.serviceId, status, {}, prevRaw, prevDisp);
|
||||||
|
};
|
||||||
|
|
||||||
|
p2(makeUp());
|
||||||
|
p2(makeDown());
|
||||||
|
p2(makeDown()); // displayed down → outage opens
|
||||||
|
expect(hc2.incidents).toHaveLength(1);
|
||||||
|
expect(hc2.incidents[0].status).toBe('open');
|
||||||
|
|
||||||
|
p2(makeUp()); // counter=1 < UP_THRESHOLD=2 → displayed still down
|
||||||
|
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
|
||||||
|
expect(hc2.incidents[0].status).toBe('open'); // NOT resolved by the blip
|
||||||
|
|
||||||
|
p2(makeUp()); // counter=2 → displayed up → incident resolves
|
||||||
|
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
|
||||||
|
expect(hc2.incidents[0].status).toBe('resolved');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('legacy direct callers (no displayed state) keep raw transition semantics', () => {
|
||||||
|
hc.currentStatus.set('svc1', { status: 'up' });
|
||||||
|
const status = { status: 'down', timestamp: new Date().toISOString(), responseTime: 100 };
|
||||||
|
hc.checkForIncidents('svc1', status, {}); // 4-arg call, no previousDisplayed
|
||||||
|
expect(hc.incidents).toHaveLength(1);
|
||||||
|
expect(hc.incidents[0].type).toBe('outage');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('slow-response detection still fires per-probe regardless of hysteresis', () => {
|
||||||
|
const slowUp = makeUp();
|
||||||
|
slowUp.responseTime = 6000;
|
||||||
|
probe(slowUp, { slowResponseThreshold: 5000 });
|
||||||
|
expect(hc.incidents.some(i => i.type === 'slow-response')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -204,15 +204,22 @@ describe('HealthChecker', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('opens and resolves an outage incident across real checkService transitions', async () => {
|
it('opens and resolves an outage incident across real checkService transitions', async () => {
|
||||||
|
// DC-090: incidents follow the DISPLAYED (post-hysteresis) status.
|
||||||
|
// DOWN_THRESHOLD defaults to 2, so it takes two consecutive failed
|
||||||
|
// probes to flip displayed down and open the outage; one up probe
|
||||||
|
// (UP_THRESHOLD=1) resolves it.
|
||||||
healthChecker._doRequest = jest.fn()
|
healthChecker._doRequest = jest.fn()
|
||||||
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
|
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
|
||||||
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
|
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
|
||||||
|
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
|
||||||
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });
|
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });
|
||||||
|
|
||||||
const config = { url: 'http://test.local' };
|
const config = { url: 'http://test.local' };
|
||||||
await healthChecker.checkService('svc1', config);
|
await healthChecker.checkService('svc1', config);
|
||||||
await healthChecker.checkService('svc1', config);
|
await healthChecker.checkService('svc1', config);
|
||||||
|
expect(healthChecker.incidents).toHaveLength(0); // one down alone: suppressed blip
|
||||||
|
|
||||||
|
await healthChecker.checkService('svc1', config); // second down flips displayed → open
|
||||||
expect(healthChecker.incidents).toHaveLength(1);
|
expect(healthChecker.incidents).toHaveLength(1);
|
||||||
expect(healthChecker.incidents[0]).toMatchObject({
|
expect(healthChecker.incidents[0]).toMatchObject({
|
||||||
serviceId: 'svc1',
|
serviceId: 'svc1',
|
||||||
@@ -220,7 +227,7 @@ describe('HealthChecker', () => {
|
|||||||
status: 'open'
|
status: 'open'
|
||||||
});
|
});
|
||||||
|
|
||||||
await healthChecker.checkService('svc1', config);
|
await healthChecker.checkService('svc1', config); // up resolves
|
||||||
expect(healthChecker.incidents[0].status).toBe('resolved');
|
expect(healthChecker.incidents[0].status).toBe('resolved');
|
||||||
expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
|
expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
|
||||||
});
|
});
|
||||||
@@ -591,6 +598,113 @@ describe('HealthChecker', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('DC-088: removeService generation tombstones + incident closure', () => {
|
||||||
|
it('does not leak a serviceGenerations entry and records a tombstone', () => {
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||||
|
expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
|
||||||
|
expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
|
||||||
|
const tomb = healthChecker.removedGenerations.get('svc1');
|
||||||
|
expect(tomb).toBeDefined();
|
||||||
|
expect(tomb.generation).toBeGreaterThan(0);
|
||||||
|
expect(tomb.removedAt).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('re-added service gets a strictly higher generation (no ABA)', () => {
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||||
|
const gen1 = healthChecker.serviceGenerations.get('svc1');
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local/v2' });
|
||||||
|
|
||||||
|
const gen2 = healthChecker.serviceGenerations.get('svc1');
|
||||||
|
expect(gen2).toBeGreaterThan(gen1);
|
||||||
|
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('closes open incidents for the removed service as resolved', () => {
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.incidents.push({
|
||||||
|
id: 'incident-test-1',
|
||||||
|
serviceId: 'svc1',
|
||||||
|
type: 'outage',
|
||||||
|
status: 'open',
|
||||||
|
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||||
|
});
|
||||||
|
healthChecker.incidents.push({
|
||||||
|
id: 'incident-other',
|
||||||
|
serviceId: 'svc2',
|
||||||
|
type: 'outage',
|
||||||
|
status: 'open',
|
||||||
|
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||||
|
});
|
||||||
|
const resolvedSpy = jest.fn();
|
||||||
|
healthChecker.on('incident-resolved', resolvedSpy);
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
|
||||||
|
const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
|
||||||
|
expect(closed.status).toBe('resolved');
|
||||||
|
expect(closed.resolvedBy).toBe('service-removed');
|
||||||
|
expect(closed.resolvedAt).toBeDefined();
|
||||||
|
expect(closed.duration).toBeGreaterThan(0);
|
||||||
|
expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
|
||||||
|
expect(resolvedSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('in-flight probe captured before removal is discarded via tombstone', async () => {
|
||||||
|
let resolveProbe;
|
||||||
|
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||||
|
healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
|
||||||
|
resolveProbe = resolve;
|
||||||
|
}));
|
||||||
|
|
||||||
|
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
|
||||||
|
await pending;
|
||||||
|
|
||||||
|
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a rejected in-flight probe after removal does not re-create failure state', async () => {
|
||||||
|
let rejectProbe;
|
||||||
|
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||||
|
healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
|
||||||
|
rejectProbe = reject;
|
||||||
|
}));
|
||||||
|
|
||||||
|
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
rejectProbe(new Error('late failure'));
|
||||||
|
await pending;
|
||||||
|
|
||||||
|
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sweeps expired tombstones in cleanupHistory', () => {
|
||||||
|
healthChecker.removedGenerations.set('svc1', {
|
||||||
|
generation: 1,
|
||||||
|
removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
|
||||||
|
});
|
||||||
|
healthChecker.removedGenerations.set('svc2', {
|
||||||
|
generation: 2,
|
||||||
|
removedAt: Date.now() // fresh
|
||||||
|
});
|
||||||
|
|
||||||
|
healthChecker.cleanupHistory();
|
||||||
|
|
||||||
|
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('cleanupHistory', () => {
|
describe('cleanupHistory', () => {
|
||||||
it('removes entries older than retention period', () => {
|
it('removes entries older than retention period', () => {
|
||||||
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
|
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ const platformPaths = require('../../platform-paths');
|
|||||||
const { createUserStore } = require('../../src/security/user-store');
|
const { createUserStore } = require('../../src/security/user-store');
|
||||||
const { createInviteStore } = require('../../src/security/invite-store');
|
const { createInviteStore } = require('../../src/security/invite-store');
|
||||||
const emailSender = require('../../src/auth/providers/email-sender');
|
const emailSender = require('../../src/auth/providers/email-sender');
|
||||||
|
const AuthProvider = require('../../src/auth/providers/base');
|
||||||
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
|
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
|
||||||
const { ok, successMessage } = require('../../src/utils/responses');
|
const { ok, successMessage } = require('../../src/utils/responses');
|
||||||
|
|
||||||
@@ -254,7 +255,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
|||||||
// server log on every unconfigured-install invite.
|
// server log on every unconfigured-install invite.
|
||||||
log.warn && log.warn('auth-invite-send',
|
log.warn && log.warn('auth-invite-send',
|
||||||
'invite send skipped: SMTP not configured (operator opted in)',
|
'invite send skipped: SMTP not configured (operator opted in)',
|
||||||
{ inviteId: issued.id, email: issued.email });
|
{ inviteId: issued.id, email: AuthProvider.maskEmail(issued.email) || '[unmaskable-email]' });
|
||||||
deliveredVia = 'failed';
|
deliveredVia = 'failed';
|
||||||
}
|
}
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
@@ -369,7 +370,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
|||||||
|
|
||||||
log.info && log.info('auth', 'invite accepted, user created', {
|
log.info && log.info('auth', 'invite accepted, user created', {
|
||||||
userId: userResult.user.id,
|
userId: userResult.user.id,
|
||||||
email: userResult.user.email,
|
email: AuthProvider.maskEmail(userResult.user.email) || '[unmaskable-email]',
|
||||||
role: userResult.user.role,
|
role: userResult.user.role,
|
||||||
inviteId: invite.id,
|
inviteId: invite.id,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -32,6 +32,10 @@ const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10
|
|||||||
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
||||||
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
||||||
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
||||||
|
// DC-088: how long a removal tombstone outlives the removal itself. Only needs
|
||||||
|
// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
|
||||||
|
// swept by cleanupHistory so removed services cannot accumulate map entries.
|
||||||
|
const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
|
||||||
|
|
||||||
// DC-086: hysteresis thresholds for badge display.
|
// DC-086: hysteresis thresholds for badge display.
|
||||||
// The raw probe result can flap on a single transient blip (Caddy reload,
|
// The raw probe result can flap on a single transient blip (Caddy reload,
|
||||||
@@ -72,6 +76,14 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
||||||
// Invalidate probe completions that race with removal/reconfiguration.
|
// Invalidate probe completions that race with removal/reconfiguration.
|
||||||
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
||||||
|
// DC-088: monotonically increasing sequence so generation numbers can never
|
||||||
|
// repeat across remove -> re-add cycles (prevents ABA on the stale check).
|
||||||
|
this.generationSeq = 0;
|
||||||
|
// DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
|
||||||
|
// serviceGenerations means the service is (re)configured; a tombstone with a
|
||||||
|
// HIGHER generation than the captured one marks the capture as stale. Entry
|
||||||
|
// is deleted when the service is removed, so the live map cannot leak.
|
||||||
|
this.removedGenerations = new Map();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -139,6 +151,21 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.cleanupHistory();
|
this.cleanupHistory();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DC-088: true when a probe's captured generation no longer matches the
|
||||||
|
* service's current configuration state. A live serviceGenerations entry
|
||||||
|
* must match exactly. With no live entry the service was never configured
|
||||||
|
* in this process (disk-loaded / direct callers) — stale only if a removal
|
||||||
|
* tombstone with a HIGHER generation exists.
|
||||||
|
*/
|
||||||
|
_isStaleCapture(serviceId, generation) {
|
||||||
|
if (this.serviceGenerations.has(serviceId)) {
|
||||||
|
return this.serviceGenerations.get(serviceId) !== generation;
|
||||||
|
}
|
||||||
|
const tomb = this.removedGenerations.get(serviceId);
|
||||||
|
return Boolean(tomb && tomb.generation > generation);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check a single service
|
* Check a single service
|
||||||
*/
|
*/
|
||||||
@@ -160,7 +187,7 @@ class HealthChecker extends EventEmitter {
|
|||||||
details: result.details
|
details: result.details
|
||||||
};
|
};
|
||||||
|
|
||||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
if (this._isStaleCapture(serviceId, generation)) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,16 +199,14 @@ class HealthChecker extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const previousStatus = this.currentStatus.get(serviceId);
|
const previousStatus = this.currentStatus.get(serviceId);
|
||||||
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||||
this.recordStatus(serviceId, status);
|
this.recordStatus(serviceId, status);
|
||||||
this.checkForIncidents(serviceId, status, config, previousStatus);
|
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
||||||
|
|
||||||
return status;
|
return status;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const responseTime = Date.now() - startTime;
|
const responseTime = Date.now() - startTime;
|
||||||
|
|
||||||
// Increment failure count for backoff
|
|
||||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
|
||||||
|
|
||||||
const status = {
|
const status = {
|
||||||
serviceId,
|
serviceId,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
@@ -190,13 +215,18 @@ class HealthChecker extends EventEmitter {
|
|||||||
error: error.message
|
error: error.message
|
||||||
};
|
};
|
||||||
|
|
||||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
if (this._isStaleCapture(serviceId, generation)) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Increment failure count for backoff — only after the result is known
|
||||||
|
// to be non-stale, so a removed service cannot re-create map entries.
|
||||||
|
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||||
|
|
||||||
const previousStatus = this.currentStatus.get(serviceId);
|
const previousStatus = this.currentStatus.get(serviceId);
|
||||||
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||||
this.recordStatus(serviceId, status);
|
this.recordStatus(serviceId, status);
|
||||||
this.checkForIncidents(serviceId, status, config, previousStatus);
|
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
||||||
|
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
@@ -425,10 +455,27 @@ class HealthChecker extends EventEmitter {
|
|||||||
/**
|
/**
|
||||||
* Check for incidents (downtime, slow response, etc.)
|
* Check for incidents (downtime, slow response, etc.)
|
||||||
*/
|
*/
|
||||||
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId)) {
|
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
||||||
|
|
||||||
// Check for status change (up -> down or down -> up)
|
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
||||||
if (previous && previous.status !== status.status) {
|
// the same signal that flips the dashboard badge. A single raw "down"
|
||||||
|
// blip that hysteresis suppresses must not open a critical outage
|
||||||
|
// incident (and a suppressed blip must not resolve a real one). When the
|
||||||
|
// caller supplies the pre-probe displayed state (checkService always
|
||||||
|
// does), transitions are evaluated displayed-vs-displayed using the
|
||||||
|
// post-recordStatus state in this.displayedStatus. Direct callers with
|
||||||
|
// no hysteresis state (previousDisplayed === null) keep the legacy
|
||||||
|
// raw-probe transition semantics.
|
||||||
|
if (previousDisplayed) {
|
||||||
|
const displayed = this.displayedStatus.get(serviceId);
|
||||||
|
if (displayed && displayed.status !== previousDisplayed.status) {
|
||||||
|
if (displayed.status === 'down') {
|
||||||
|
this.createIncident(serviceId, 'outage', 'Service is down', displayed);
|
||||||
|
} else if (displayed.status === 'up') {
|
||||||
|
this.resolveIncident(serviceId, 'outage', displayed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (previous && previous.status !== status.status) {
|
||||||
if (status.status === 'down') {
|
if (status.status === 'down') {
|
||||||
this.createIncident(serviceId, 'outage', 'Service is down', status);
|
this.createIncident(serviceId, 'outage', 'Service is down', status);
|
||||||
} else if (status.status === 'up') {
|
} else if (status.status === 'up') {
|
||||||
@@ -660,7 +707,13 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.config.services = {};
|
this.config.services = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
// DC-088: monotonic instance-wide sequence — a re-added service can never
|
||||||
|
// recycle a previous generation number, and any older in-flight capture is
|
||||||
|
// invalidated by definition.
|
||||||
|
this.generationSeq += 1;
|
||||||
|
this.serviceGenerations.set(serviceId, this.generationSeq);
|
||||||
|
// Re-configuration supersedes any prior removal tombstone.
|
||||||
|
this.removedGenerations.delete(serviceId);
|
||||||
this.config.services[serviceId] = {
|
this.config.services[serviceId] = {
|
||||||
enabled: config.enabled !== false,
|
enabled: config.enabled !== false,
|
||||||
name: config.name || serviceId,
|
name: config.name || serviceId,
|
||||||
@@ -683,12 +736,35 @@ class HealthChecker extends EventEmitter {
|
|||||||
* Remove service configuration
|
* Remove service configuration
|
||||||
*/
|
*/
|
||||||
removeService(serviceId) {
|
removeService(serviceId) {
|
||||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
// DC-088: tombstone the captured generation instead of leaking an entry.
|
||||||
|
// The live map entry is deleted; an in-flight probe captured BEFORE this
|
||||||
|
// point sees no live entry but a higher tombstone generation, so it is
|
||||||
|
// discarded. configureService clears the tombstone on re-add.
|
||||||
|
this.generationSeq += 1;
|
||||||
|
this.serviceGenerations.delete(serviceId);
|
||||||
|
this.removedGenerations.set(serviceId, {
|
||||||
|
generation: this.generationSeq,
|
||||||
|
removedAt: Date.now()
|
||||||
|
});
|
||||||
if (this.config.services) {
|
if (this.config.services) {
|
||||||
delete this.config.services[serviceId];
|
delete this.config.services[serviceId];
|
||||||
this.saveConfig();
|
this.saveConfig();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DC-088: open incidents for a removed service must not linger forever.
|
||||||
|
// Close them through the same resolve path a recovery would, annotated so
|
||||||
|
// history shows why (dashboard renders resolved incidents green + duration).
|
||||||
|
for (const incident of this.incidents) {
|
||||||
|
if (incident.serviceId === serviceId && incident.status === 'open') {
|
||||||
|
incident.status = 'resolved';
|
||||||
|
incident.resolvedAt = new Date().toISOString();
|
||||||
|
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
||||||
|
incident.resolvedBy = 'service-removed';
|
||||||
|
this.emit('incident-resolved', incident);
|
||||||
|
this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
this.currentStatus.delete(serviceId);
|
this.currentStatus.delete(serviceId);
|
||||||
this.displayedStatus.delete(serviceId);
|
this.displayedStatus.delete(serviceId);
|
||||||
this.consecutiveSinceChange.delete(serviceId);
|
this.consecutiveSinceChange.delete(serviceId);
|
||||||
@@ -714,6 +790,18 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DC-088: sweep expired removal tombstones. After the TTL no probe that
|
||||||
|
// captured a pre-removal generation can still be in flight (timeout is
|
||||||
|
// bounded by performHealthCheck), so the tombstone has done its job.
|
||||||
|
if (this.removedGenerations.size > 0) {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [serviceId, tomb] of this.removedGenerations) {
|
||||||
|
if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
|
||||||
|
this.removedGenerations.delete(serviceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user