Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df55677bd1 | ||
|
|
ddbea0a040 | ||
|
|
1d1cd5c95e | ||
|
|
88f1d4a414 | ||
|
|
dd1110ef52 | ||
|
|
6732a1e1df | ||
|
|
ea96abe95a | ||
|
|
3ccf66754a | ||
|
|
c71b794ccc |
@@ -237,4 +237,49 @@ describe('DC-085: link-first admin invites', () => {
|
||||
.send({ email: 'a@x.com', ttlHours: 1 });
|
||||
expect(res.body.shareText).toContain('expires in 1h');
|
||||
});
|
||||
});
|
||||
|
||||
test('DC-089: SMTP-unconfigured warn log masks the invite email (no raw PII)', async () => {
|
||||
mockEmailSender.isConfigured.mockReturnValueOnce(false);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/v1/auth/admin/invites')
|
||||
.send({ email: 'friend@example.com', role: 'operator', sendEmail: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.deliveredVia).toBe('failed');
|
||||
const warn = logCalls.find(c =>
|
||||
c.level === 'warn' && c.topic === 'auth-invite-send'
|
||||
);
|
||||
expect(warn).toBeDefined();
|
||||
// The raw address must not appear; the masked form must.
|
||||
expect(JSON.stringify(warn.meta)).not.toContain('friend@example.com');
|
||||
expect(warn.meta.email).toBe('fr****@example.com');
|
||||
});
|
||||
|
||||
test('DC-089: invite-accepted info log masks the created user email (no raw PII)', async () => {
|
||||
// Pre-authorize the email (POST /admin/users) so userStore.login doesn't
|
||||
// reject with not_authorized — bootstrap already happened in beforeEach.
|
||||
const preauth = await request(app)
|
||||
.post('/api/v1/auth/admin/users')
|
||||
.send({ email: 'newfriend@example.com' });
|
||||
expect(preauth.status).toBe(200);
|
||||
|
||||
const issue = await request(app)
|
||||
.post('/api/v1/auth/admin/invites')
|
||||
.send({ email: 'newfriend@example.com', role: 'viewer' });
|
||||
expect(issue.status).toBe(200);
|
||||
const token = issue.body.acceptUrl.match(/invites\/([^/]+)\/accept/)[1];
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/v1/auth/invites/${token}/accept`)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const info = logCalls.find(c =>
|
||||
c.level === 'info' && c.msg === 'invite accepted, user created'
|
||||
);
|
||||
expect(info).toBeDefined();
|
||||
expect(JSON.stringify(info.meta)).not.toContain('newfriend@example.com');
|
||||
expect(info.meta.email).toBe('ne****@example.com');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Regression tests for config-schema.js KNOWN_KEYS — DC-091.
|
||||
*
|
||||
* Bug: license-manager.js persists config.licenseBackup (activation
|
||||
* restore-on-restart) and src/config/migrations.js stamps config._version,
|
||||
* but neither key was in KNOWN_KEYS — so every startup logged
|
||||
* `Unknown config key "licenseBackup" / "_version" — possible typo?`
|
||||
* false positives (verified in live dashcaddy-api container logs,
|
||||
* 2026-08-22T23:53:54Z restart).
|
||||
*
|
||||
* These tests pin: (1) the live production config key set validates with
|
||||
* zero unknown-key warnings, (2) genuine typos still warn, (3) the schema
|
||||
* stays in sync with the first-party writer keys.
|
||||
*/
|
||||
|
||||
const { validateConfig } = require('../src/utilities/config-schema');
|
||||
|
||||
describe('config-schema KNOWN_KEYS vs first-party writers (DC-091)', () => {
|
||||
// Exact key set of the live production config.json (DNS2, verified
|
||||
// 2026-08-23). If a new key appears here, teach KNOWN_KEYS about it —
|
||||
// or fix the writer if it's a typo.
|
||||
const LIVE_CONFIG_KEYS = [
|
||||
'_version', 'configurationType', 'customFavicon', 'customLogo',
|
||||
'dashboardHost', 'dashboardTitle', 'dns', 'dnsServers', 'language',
|
||||
'license', 'licenseBackup', 'logoPosition', 'pylon', 'setupComplete',
|
||||
'timestamp', 'tld', 'updatedAt'
|
||||
];
|
||||
|
||||
test('live production config key set produces zero unknown-key warnings', () => {
|
||||
const config = {};
|
||||
for (const key of LIVE_CONFIG_KEYS) {
|
||||
// Minimal valid-ish values; validateConfig only cares about shape
|
||||
// for these keys, and unknown-key detection is the target here.
|
||||
config[key] = key === '_version' ? 2 : (key === 'dnsServers' ? {} : 'x');
|
||||
}
|
||||
const result = validateConfig(config);
|
||||
const unknownWarnings = result.warnings.filter((w) => w.includes('Unknown config key'));
|
||||
expect(unknownWarnings).toEqual([]);
|
||||
});
|
||||
|
||||
test('licenseBackup and _version (first-party writer keys) do not warn', () => {
|
||||
const result = validateConfig({ licenseBackup: { code: 'DC-...' }, _version: 2 });
|
||||
expect(result.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
test('genuine typos still warn (guard against over-allowing)', () => {
|
||||
const result = validateConfig({ dashboadTitle: 'typo' });
|
||||
expect(result.warnings).toEqual([
|
||||
'Unknown config key "dashboadTitle" — possible typo?'
|
||||
]);
|
||||
});
|
||||
|
||||
test('KNOWN_KEYS stays in sync with license-manager writer keys', () => {
|
||||
// license-manager writes config.licenseBackup and config.license — both
|
||||
// must be recognized. We assert via validateConfig (public surface)
|
||||
// rather than importing the private KNOWN_KEYS array.
|
||||
const result = validateConfig({ license: { code: 'DC-...' }, licenseBackup: { code: 'DC-...' } });
|
||||
expect(result.warnings.filter((w) => w.includes('Unknown config key'))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('config-schema sync guard: migrations writer', () => {
|
||||
test('_version is recognized at every migration version value', () => {
|
||||
// migrations.js bumps _version 0→1→2; the key itself must never warn.
|
||||
for (const v of [0, 1, 2, 99]) {
|
||||
const result = validateConfig({ _version: v });
|
||||
expect(result.warnings).toEqual([]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Tests for DC-090: outage incidents follow the DISPLAYED (post-hysteresis)
|
||||
* status — the same signal that flips the dashboard badge.
|
||||
*
|
||||
* - A single raw "down" blip that hysteresis suppresses opens NO outage
|
||||
* incident (the DC-089-noted raw-transition bug).
|
||||
* - A suppressed blip does not resolve a real open outage (UP_THRESHOLD=2).
|
||||
* - DOWN_THRESHOLD consecutive downs open exactly ONE outage incident.
|
||||
* - The incident payload carries the displayed snapshot, not the raw probe.
|
||||
* - Direct callers without hysteresis state keep legacy raw semantics.
|
||||
*
|
||||
* The probe() helper replicates checkService's exact call order: capture the
|
||||
* pre-probe raw + displayed state, recordStatus (updates both maps), then
|
||||
* checkForIncidents with both previous states.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
|
||||
// Use an isolated data dir so test history doesn't pollute the real one.
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dashcaddy-incpar-'));
|
||||
process.env.HEALTH_DATA_DIR = tmpDir;
|
||||
process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
|
||||
process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
|
||||
|
||||
// Module exports a singleton instance, not a class. Reset per-test state by
|
||||
// replacing the relevant maps on the singleton in beforeEach.
|
||||
const healthCheckerSingleton = require('../src/monitoring/health-checker');
|
||||
const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD;
|
||||
const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD;
|
||||
|
||||
function restoreEnv(name, value) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
|
||||
function makeUp(serviceId = 'svc1') {
|
||||
return {
|
||||
serviceId,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'up',
|
||||
responseTime: 50,
|
||||
statusCode: 200,
|
||||
message: 'Service is healthy',
|
||||
details: { headers: {}, bodyLength: 12 }
|
||||
};
|
||||
}
|
||||
|
||||
function makeDown(serviceId = 'svc1') {
|
||||
return {
|
||||
serviceId,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'down',
|
||||
responseTime: 50,
|
||||
statusCode: 500,
|
||||
message: 'fail',
|
||||
details: { headers: {}, bodyLength: 0 }
|
||||
};
|
||||
}
|
||||
|
||||
describe('DC-090: outage incidents follow the displayed (hysteresis) status', () => {
|
||||
let hc;
|
||||
let incidentCreatedSpy;
|
||||
let incidentResolvedSpy;
|
||||
|
||||
beforeEach(() => {
|
||||
healthCheckerSingleton.displayedStatus = new Map();
|
||||
healthCheckerSingleton.consecutiveSinceChange = new Map();
|
||||
healthCheckerSingleton.currentStatus = new Map();
|
||||
healthCheckerSingleton.history = {};
|
||||
healthCheckerSingleton.incidents = [];
|
||||
healthCheckerSingleton.removeAllListeners('incident-created');
|
||||
healthCheckerSingleton.removeAllListeners('incident-resolved');
|
||||
incidentCreatedSpy = jest.fn();
|
||||
incidentResolvedSpy = jest.fn();
|
||||
healthCheckerSingleton.on('incident-created', incidentCreatedSpy);
|
||||
healthCheckerSingleton.on('incident-resolved', incidentResolvedSpy);
|
||||
hc = healthCheckerSingleton;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold);
|
||||
restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Replicates checkService's record+incident sequence for one raw probe.
|
||||
function probe(status, config = {}) {
|
||||
const previousStatus = hc.currentStatus.get(status.serviceId);
|
||||
const previousDisplayed = hc.displayedStatus.get(status.serviceId) || null;
|
||||
hc.recordStatus(status.serviceId, status);
|
||||
hc.checkForIncidents(status.serviceId, status, config, previousStatus, previousDisplayed);
|
||||
}
|
||||
|
||||
test('a single down blip between two ups opens NO outage incident', () => {
|
||||
probe(makeUp()); // baseline: displayed up
|
||||
probe(makeDown()); // blip — hysteresis keeps displayed up
|
||||
probe(makeUp()); // recovered
|
||||
expect(hc.incidents).toHaveLength(0);
|
||||
expect(incidentCreatedSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('DOWN_THRESHOLD consecutive downs open exactly one outage incident (critical)', () => {
|
||||
probe(makeUp());
|
||||
probe(makeDown()); // counter=1, displayed still up
|
||||
probe(makeDown()); // counter=2 → displayed flips down → incident
|
||||
expect(hc.incidents).toHaveLength(1);
|
||||
const incident = hc.incidents[0];
|
||||
expect(incident.type).toBe('outage');
|
||||
expect(incident.severity).toBe('critical');
|
||||
expect(incident.status).toBe('open');
|
||||
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
probe(makeDown()); // still down — no new transition, no second incident
|
||||
expect(hc.incidents).toHaveLength(1);
|
||||
expect(incident.occurrences).toBe(1); // occurrences count displayed flips, not raw probes
|
||||
expect(incidentCreatedSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('the outage incident payload carries the displayed snapshot, not the raw blip', () => {
|
||||
probe(makeUp());
|
||||
const blip = makeDown();
|
||||
blip.statusCode = 599;
|
||||
probe(blip); // suppressed blip — must not appear in any incident
|
||||
probe(makeDown()); // flip
|
||||
expect(hc.incidents).toHaveLength(1);
|
||||
// The incident's details snapshot is the probe that FLIPPED the displayed
|
||||
// state (the second down), not the earlier suppressed blip.
|
||||
expect(hc.incidents[0].details.statusCode).not.toBe(599);
|
||||
});
|
||||
|
||||
test('a suppressed up blip does not resolve a real open outage (UP_THRESHOLD=2)', () => {
|
||||
process.env.HEALTH_UP_THRESHOLD = '2';
|
||||
jest.resetModules();
|
||||
const hc2 = require('../src/monitoring/health-checker');
|
||||
hc2.displayedStatus = new Map();
|
||||
hc2.consecutiveSinceChange = new Map();
|
||||
hc2.currentStatus = new Map();
|
||||
hc2.history = {};
|
||||
hc2.incidents = [];
|
||||
hc2.removeAllListeners('incident-created');
|
||||
hc2.removeAllListeners('incident-resolved');
|
||||
|
||||
const p2 = (status) => {
|
||||
const prevRaw = hc2.currentStatus.get(status.serviceId);
|
||||
const prevDisp = hc2.displayedStatus.get(status.serviceId) || null;
|
||||
hc2.recordStatus(status.serviceId, status);
|
||||
hc2.checkForIncidents(status.serviceId, status, {}, prevRaw, prevDisp);
|
||||
};
|
||||
|
||||
p2(makeUp());
|
||||
p2(makeDown());
|
||||
p2(makeDown()); // displayed down → outage opens
|
||||
expect(hc2.incidents).toHaveLength(1);
|
||||
expect(hc2.incidents[0].status).toBe('open');
|
||||
|
||||
p2(makeUp()); // counter=1 < UP_THRESHOLD=2 → displayed still down
|
||||
expect(hc2.displayedStatus.get('svc1').status).toBe('down');
|
||||
expect(hc2.incidents[0].status).toBe('open'); // NOT resolved by the blip
|
||||
|
||||
p2(makeUp()); // counter=2 → displayed up → incident resolves
|
||||
expect(hc2.displayedStatus.get('svc1').status).toBe('up');
|
||||
expect(hc2.incidents[0].status).toBe('resolved');
|
||||
});
|
||||
|
||||
test('legacy direct callers (no displayed state) keep raw transition semantics', () => {
|
||||
hc.currentStatus.set('svc1', { status: 'up' });
|
||||
const status = { status: 'down', timestamp: new Date().toISOString(), responseTime: 100 };
|
||||
hc.checkForIncidents('svc1', status, {}); // 4-arg call, no previousDisplayed
|
||||
expect(hc.incidents).toHaveLength(1);
|
||||
expect(hc.incidents[0].type).toBe('outage');
|
||||
});
|
||||
|
||||
test('slow-response detection still fires per-probe regardless of hysteresis', () => {
|
||||
const slowUp = makeUp();
|
||||
slowUp.responseTime = 6000;
|
||||
probe(slowUp, { slowResponseThreshold: 5000 });
|
||||
expect(hc.incidents.some(i => i.type === 'slow-response')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -204,15 +204,22 @@ describe('HealthChecker', () => {
|
||||
});
|
||||
|
||||
it('opens and resolves an outage incident across real checkService transitions', async () => {
|
||||
// DC-090: incidents follow the DISPLAYED (post-hysteresis) status.
|
||||
// DOWN_THRESHOLD defaults to 2, so it takes two consecutive failed
|
||||
// probes to flip displayed down and open the outage; one up probe
|
||||
// (UP_THRESHOLD=1) resolves it.
|
||||
healthChecker._doRequest = jest.fn()
|
||||
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
|
||||
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
|
||||
.mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
|
||||
.mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });
|
||||
|
||||
const config = { url: 'http://test.local' };
|
||||
await healthChecker.checkService('svc1', config);
|
||||
await healthChecker.checkService('svc1', config);
|
||||
expect(healthChecker.incidents).toHaveLength(0); // one down alone: suppressed blip
|
||||
|
||||
await healthChecker.checkService('svc1', config); // second down flips displayed → open
|
||||
expect(healthChecker.incidents).toHaveLength(1);
|
||||
expect(healthChecker.incidents[0]).toMatchObject({
|
||||
serviceId: 'svc1',
|
||||
@@ -220,7 +227,7 @@ describe('HealthChecker', () => {
|
||||
status: 'open'
|
||||
});
|
||||
|
||||
await healthChecker.checkService('svc1', config);
|
||||
await healthChecker.checkService('svc1', config); // up resolves
|
||||
expect(healthChecker.incidents[0].status).toBe('resolved');
|
||||
expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
|
||||
});
|
||||
@@ -591,6 +598,113 @@ describe('HealthChecker', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('DC-088: removeService generation tombstones + incident closure', () => {
|
||||
it('does not leak a serviceGenerations entry and records a tombstone', () => {
|
||||
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||
expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);
|
||||
|
||||
healthChecker.removeService('svc1');
|
||||
|
||||
expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
|
||||
const tomb = healthChecker.removedGenerations.get('svc1');
|
||||
expect(tomb).toBeDefined();
|
||||
expect(tomb.generation).toBeGreaterThan(0);
|
||||
expect(tomb.removedAt).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('re-added service gets a strictly higher generation (no ABA)', () => {
|
||||
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||
const gen1 = healthChecker.serviceGenerations.get('svc1');
|
||||
|
||||
healthChecker.removeService('svc1');
|
||||
healthChecker.configureService('svc1', { url: 'http://test.local/v2' });
|
||||
|
||||
const gen2 = healthChecker.serviceGenerations.get('svc1');
|
||||
expect(gen2).toBeGreaterThan(gen1);
|
||||
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||
});
|
||||
|
||||
it('closes open incidents for the removed service as resolved', () => {
|
||||
healthChecker.saveConfig = jest.fn();
|
||||
healthChecker.incidents.push({
|
||||
id: 'incident-test-1',
|
||||
serviceId: 'svc1',
|
||||
type: 'outage',
|
||||
status: 'open',
|
||||
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||
});
|
||||
healthChecker.incidents.push({
|
||||
id: 'incident-other',
|
||||
serviceId: 'svc2',
|
||||
type: 'outage',
|
||||
status: 'open',
|
||||
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||
});
|
||||
const resolvedSpy = jest.fn();
|
||||
healthChecker.on('incident-resolved', resolvedSpy);
|
||||
|
||||
healthChecker.removeService('svc1');
|
||||
|
||||
const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
|
||||
expect(closed.status).toBe('resolved');
|
||||
expect(closed.resolvedBy).toBe('service-removed');
|
||||
expect(closed.resolvedAt).toBeDefined();
|
||||
expect(closed.duration).toBeGreaterThan(0);
|
||||
expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
|
||||
expect(resolvedSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('in-flight probe captured before removal is discarded via tombstone', async () => {
|
||||
let resolveProbe;
|
||||
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||
healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
|
||||
resolveProbe = resolve;
|
||||
}));
|
||||
|
||||
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||
healthChecker.saveConfig = jest.fn();
|
||||
healthChecker.removeService('svc1');
|
||||
resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
|
||||
await pending;
|
||||
|
||||
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||
});
|
||||
|
||||
it('a rejected in-flight probe after removal does not re-create failure state', async () => {
|
||||
let rejectProbe;
|
||||
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||
healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
|
||||
rejectProbe = reject;
|
||||
}));
|
||||
|
||||
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||
healthChecker.saveConfig = jest.fn();
|
||||
healthChecker.removeService('svc1');
|
||||
rejectProbe(new Error('late failure'));
|
||||
await pending;
|
||||
|
||||
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||
});
|
||||
|
||||
it('sweeps expired tombstones in cleanupHistory', () => {
|
||||
healthChecker.removedGenerations.set('svc1', {
|
||||
generation: 1,
|
||||
removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
|
||||
});
|
||||
healthChecker.removedGenerations.set('svc2', {
|
||||
generation: 2,
|
||||
removedAt: Date.now() // fresh
|
||||
});
|
||||
|
||||
healthChecker.cleanupHistory();
|
||||
|
||||
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||
expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupHistory', () => {
|
||||
it('removes entries older than retention period', () => {
|
||||
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
|
||||
|
||||
@@ -29,6 +29,7 @@ const platformPaths = require('../../platform-paths');
|
||||
const { createUserStore } = require('../../src/security/user-store');
|
||||
const { createInviteStore } = require('../../src/security/invite-store');
|
||||
const emailSender = require('../../src/auth/providers/email-sender');
|
||||
const AuthProvider = require('../../src/auth/providers/base');
|
||||
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
|
||||
const { ok, successMessage } = require('../../src/utils/responses');
|
||||
|
||||
@@ -254,7 +255,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
||||
// server log on every unconfigured-install invite.
|
||||
log.warn && log.warn('auth-invite-send',
|
||||
'invite send skipped: SMTP not configured (operator opted in)',
|
||||
{ inviteId: issued.id, email: issued.email });
|
||||
{ inviteId: issued.id, email: AuthProvider.maskEmail(issued.email) || '[unmaskable-email]' });
|
||||
deliveredVia = 'failed';
|
||||
}
|
||||
} catch (sendErr) {
|
||||
@@ -369,7 +370,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
||||
|
||||
log.info && log.info('auth', 'invite accepted, user created', {
|
||||
userId: userResult.user.id,
|
||||
email: userResult.user.email,
|
||||
email: AuthProvider.maskEmail(userResult.user.email) || '[unmaskable-email]',
|
||||
role: userResult.user.role,
|
||||
inviteId: invite.id,
|
||||
});
|
||||
|
||||
@@ -32,6 +32,10 @@ const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10
|
||||
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
||||
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
||||
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
||||
// DC-088: how long a removal tombstone outlives the removal itself. Only needs
|
||||
// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
|
||||
// swept by cleanupHistory so removed services cannot accumulate map entries.
|
||||
const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
|
||||
|
||||
// DC-086: hysteresis thresholds for badge display.
|
||||
// The raw probe result can flap on a single transient blip (Caddy reload,
|
||||
@@ -72,6 +76,14 @@ class HealthChecker extends EventEmitter {
|
||||
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
||||
// Invalidate probe completions that race with removal/reconfiguration.
|
||||
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
||||
// DC-088: monotonically increasing sequence so generation numbers can never
|
||||
// repeat across remove -> re-add cycles (prevents ABA on the stale check).
|
||||
this.generationSeq = 0;
|
||||
// DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
|
||||
// serviceGenerations means the service is (re)configured; a tombstone with a
|
||||
// HIGHER generation than the captured one marks the capture as stale. Entry
|
||||
// is deleted when the service is removed, so the live map cannot leak.
|
||||
this.removedGenerations = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -139,6 +151,21 @@ class HealthChecker extends EventEmitter {
|
||||
this.cleanupHistory();
|
||||
}
|
||||
|
||||
/**
|
||||
* DC-088: true when a probe's captured generation no longer matches the
|
||||
* service's current configuration state. A live serviceGenerations entry
|
||||
* must match exactly. With no live entry the service was never configured
|
||||
* in this process (disk-loaded / direct callers) — stale only if a removal
|
||||
* tombstone with a HIGHER generation exists.
|
||||
*/
|
||||
_isStaleCapture(serviceId, generation) {
|
||||
if (this.serviceGenerations.has(serviceId)) {
|
||||
return this.serviceGenerations.get(serviceId) !== generation;
|
||||
}
|
||||
const tomb = this.removedGenerations.get(serviceId);
|
||||
return Boolean(tomb && tomb.generation > generation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a single service
|
||||
*/
|
||||
@@ -160,7 +187,7 @@ class HealthChecker extends EventEmitter {
|
||||
details: result.details
|
||||
};
|
||||
|
||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
||||
if (this._isStaleCapture(serviceId, generation)) {
|
||||
return status;
|
||||
}
|
||||
|
||||
@@ -172,16 +199,14 @@ class HealthChecker extends EventEmitter {
|
||||
}
|
||||
|
||||
const previousStatus = this.currentStatus.get(serviceId);
|
||||
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||
this.recordStatus(serviceId, status);
|
||||
this.checkForIncidents(serviceId, status, config, previousStatus);
|
||||
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
||||
|
||||
return status;
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
// Increment failure count for backoff
|
||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||
|
||||
const status = {
|
||||
serviceId,
|
||||
timestamp: new Date().toISOString(),
|
||||
@@ -190,13 +215,18 @@ class HealthChecker extends EventEmitter {
|
||||
error: error.message
|
||||
};
|
||||
|
||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
||||
if (this._isStaleCapture(serviceId, generation)) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// Increment failure count for backoff — only after the result is known
|
||||
// to be non-stale, so a removed service cannot re-create map entries.
|
||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||
|
||||
const previousStatus = this.currentStatus.get(serviceId);
|
||||
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||
this.recordStatus(serviceId, status);
|
||||
this.checkForIncidents(serviceId, status, config, previousStatus);
|
||||
this.checkForIncidents(serviceId, status, config, previousStatus, previousDisplayed);
|
||||
|
||||
return status;
|
||||
}
|
||||
@@ -425,10 +455,27 @@ class HealthChecker extends EventEmitter {
|
||||
/**
|
||||
* Check for incidents (downtime, slow response, etc.)
|
||||
*/
|
||||
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId)) {
|
||||
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
||||
|
||||
// Check for status change (up -> down or down -> up)
|
||||
if (previous && previous.status !== status.status) {
|
||||
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
||||
// the same signal that flips the dashboard badge. A single raw "down"
|
||||
// blip that hysteresis suppresses must not open a critical outage
|
||||
// incident (and a suppressed blip must not resolve a real one). When the
|
||||
// caller supplies the pre-probe displayed state (checkService always
|
||||
// does), transitions are evaluated displayed-vs-displayed using the
|
||||
// post-recordStatus state in this.displayedStatus. Direct callers with
|
||||
// no hysteresis state (previousDisplayed === null) keep the legacy
|
||||
// raw-probe transition semantics.
|
||||
if (previousDisplayed) {
|
||||
const displayed = this.displayedStatus.get(serviceId);
|
||||
if (displayed && displayed.status !== previousDisplayed.status) {
|
||||
if (displayed.status === 'down') {
|
||||
this.createIncident(serviceId, 'outage', 'Service is down', displayed);
|
||||
} else if (displayed.status === 'up') {
|
||||
this.resolveIncident(serviceId, 'outage', displayed);
|
||||
}
|
||||
}
|
||||
} else if (previous && previous.status !== status.status) {
|
||||
if (status.status === 'down') {
|
||||
this.createIncident(serviceId, 'outage', 'Service is down', status);
|
||||
} else if (status.status === 'up') {
|
||||
@@ -660,7 +707,13 @@ class HealthChecker extends EventEmitter {
|
||||
this.config.services = {};
|
||||
}
|
||||
|
||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
||||
// DC-088: monotonic instance-wide sequence — a re-added service can never
|
||||
// recycle a previous generation number, and any older in-flight capture is
|
||||
// invalidated by definition.
|
||||
this.generationSeq += 1;
|
||||
this.serviceGenerations.set(serviceId, this.generationSeq);
|
||||
// Re-configuration supersedes any prior removal tombstone.
|
||||
this.removedGenerations.delete(serviceId);
|
||||
this.config.services[serviceId] = {
|
||||
enabled: config.enabled !== false,
|
||||
name: config.name || serviceId,
|
||||
@@ -683,12 +736,35 @@ class HealthChecker extends EventEmitter {
|
||||
* Remove service configuration
|
||||
*/
|
||||
removeService(serviceId) {
|
||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
||||
// DC-088: tombstone the captured generation instead of leaking an entry.
|
||||
// The live map entry is deleted; an in-flight probe captured BEFORE this
|
||||
// point sees no live entry but a higher tombstone generation, so it is
|
||||
// discarded. configureService clears the tombstone on re-add.
|
||||
this.generationSeq += 1;
|
||||
this.serviceGenerations.delete(serviceId);
|
||||
this.removedGenerations.set(serviceId, {
|
||||
generation: this.generationSeq,
|
||||
removedAt: Date.now()
|
||||
});
|
||||
if (this.config.services) {
|
||||
delete this.config.services[serviceId];
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
// DC-088: open incidents for a removed service must not linger forever.
|
||||
// Close them through the same resolve path a recovery would, annotated so
|
||||
// history shows why (dashboard renders resolved incidents green + duration).
|
||||
for (const incident of this.incidents) {
|
||||
if (incident.serviceId === serviceId && incident.status === 'open') {
|
||||
incident.status = 'resolved';
|
||||
incident.resolvedAt = new Date().toISOString();
|
||||
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
||||
incident.resolvedBy = 'service-removed';
|
||||
this.emit('incident-resolved', incident);
|
||||
this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.currentStatus.delete(serviceId);
|
||||
this.displayedStatus.delete(serviceId);
|
||||
this.consecutiveSinceChange.delete(serviceId);
|
||||
@@ -714,6 +790,18 @@ class HealthChecker extends EventEmitter {
|
||||
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
||||
}
|
||||
}
|
||||
|
||||
// DC-088: sweep expired removal tombstones. After the TTL no probe that
|
||||
// captured a pre-removal generation can still be in flight (timeout is
|
||||
// bounded by performHealthCheck), so the tombstone has done its job.
|
||||
if (this.removedGenerations.size > 0) {
|
||||
const now = Date.now();
|
||||
for (const [serviceId, tomb] of this.removedGenerations) {
|
||||
if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
|
||||
this.removedGenerations.delete(serviceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,7 +14,11 @@ const KNOWN_KEYS = [
|
||||
'configurationType', 'defaults', 'customLogo', 'customFavicon',
|
||||
'dashboardTitle', 'tailscale', 'license', 'skipped',
|
||||
'routingMode', 'domain', 'email', 'defaultIP', 'pylon',
|
||||
'customLogoDark', 'customLogoLight', 'language'
|
||||
'customLogoDark', 'customLogoLight', 'language',
|
||||
// license-manager.js persists the last activation to config.licenseBackup
|
||||
// (restore-on-restart path); src/config/migrations.js stamps _version.
|
||||
// Both are first-party writes — see DC-091.
|
||||
'licenseBackup', '_version'
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user