Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6732a1e1df | ||
|
|
ea96abe95a | ||
|
|
3ccf66754a | ||
|
|
c71b794ccc | ||
|
|
f2285a2550 | ||
|
|
54a1df5ac4 | ||
|
|
eb546bf468 |
@@ -237,4 +237,49 @@ describe('DC-085: link-first admin invites', () => {
|
|||||||
.send({ email: 'a@x.com', ttlHours: 1 });
|
.send({ email: 'a@x.com', ttlHours: 1 });
|
||||||
expect(res.body.shareText).toContain('expires in 1h');
|
expect(res.body.shareText).toContain('expires in 1h');
|
||||||
});
|
});
|
||||||
});
|
|
||||||
|
test('DC-089: SMTP-unconfigured warn log masks the invite email (no raw PII)', async () => {
|
||||||
|
mockEmailSender.isConfigured.mockReturnValueOnce(false);
|
||||||
|
|
||||||
|
const res = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/invites')
|
||||||
|
.send({ email: 'friend@example.com', role: 'operator', sendEmail: true });
|
||||||
|
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.body.deliveredVia).toBe('failed');
|
||||||
|
const warn = logCalls.find(c =>
|
||||||
|
c.level === 'warn' && c.topic === 'auth-invite-send'
|
||||||
|
);
|
||||||
|
expect(warn).toBeDefined();
|
||||||
|
// The raw address must not appear; the masked form must.
|
||||||
|
expect(JSON.stringify(warn.meta)).not.toContain('friend@example.com');
|
||||||
|
expect(warn.meta.email).toBe('fr****@example.com');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('DC-089: invite-accepted info log masks the created user email (no raw PII)', async () => {
|
||||||
|
// Pre-authorize the email (POST /admin/users) so userStore.login doesn't
|
||||||
|
// reject with not_authorized — bootstrap already happened in beforeEach.
|
||||||
|
const preauth = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/users')
|
||||||
|
.send({ email: 'newfriend@example.com' });
|
||||||
|
expect(preauth.status).toBe(200);
|
||||||
|
|
||||||
|
const issue = await request(app)
|
||||||
|
.post('/api/v1/auth/admin/invites')
|
||||||
|
.send({ email: 'newfriend@example.com', role: 'viewer' });
|
||||||
|
expect(issue.status).toBe(200);
|
||||||
|
const token = issue.body.acceptUrl.match(/invites\/([^/]+)\/accept/)[1];
|
||||||
|
|
||||||
|
const res = await request(app)
|
||||||
|
.post(`/api/v1/auth/invites/${token}/accept`)
|
||||||
|
.send({});
|
||||||
|
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
const info = logCalls.find(c =>
|
||||||
|
c.level === 'info' && c.msg === 'invite accepted, user created'
|
||||||
|
);
|
||||||
|
expect(info).toBeDefined();
|
||||||
|
expect(JSON.stringify(info.meta)).not.toContain('newfriend@example.com');
|
||||||
|
expect(info.meta.email).toBe('ne****@example.com');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -591,6 +591,113 @@ describe('HealthChecker', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('DC-088: removeService generation tombstones + incident closure', () => {
|
||||||
|
it('does not leak a serviceGenerations entry and records a tombstone', () => {
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||||
|
expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
|
||||||
|
expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
|
||||||
|
const tomb = healthChecker.removedGenerations.get('svc1');
|
||||||
|
expect(tomb).toBeDefined();
|
||||||
|
expect(tomb.generation).toBeGreaterThan(0);
|
||||||
|
expect(tomb.removedAt).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('re-added service gets a strictly higher generation (no ABA)', () => {
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local' });
|
||||||
|
const gen1 = healthChecker.serviceGenerations.get('svc1');
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
healthChecker.configureService('svc1', { url: 'http://test.local/v2' });
|
||||||
|
|
||||||
|
const gen2 = healthChecker.serviceGenerations.get('svc1');
|
||||||
|
expect(gen2).toBeGreaterThan(gen1);
|
||||||
|
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('closes open incidents for the removed service as resolved', () => {
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.incidents.push({
|
||||||
|
id: 'incident-test-1',
|
||||||
|
serviceId: 'svc1',
|
||||||
|
type: 'outage',
|
||||||
|
status: 'open',
|
||||||
|
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||||
|
});
|
||||||
|
healthChecker.incidents.push({
|
||||||
|
id: 'incident-other',
|
||||||
|
serviceId: 'svc2',
|
||||||
|
type: 'outage',
|
||||||
|
status: 'open',
|
||||||
|
createdAt: new Date(Date.now() - 60_000).toISOString()
|
||||||
|
});
|
||||||
|
const resolvedSpy = jest.fn();
|
||||||
|
healthChecker.on('incident-resolved', resolvedSpy);
|
||||||
|
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
|
||||||
|
const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
|
||||||
|
expect(closed.status).toBe('resolved');
|
||||||
|
expect(closed.resolvedBy).toBe('service-removed');
|
||||||
|
expect(closed.resolvedAt).toBeDefined();
|
||||||
|
expect(closed.duration).toBeGreaterThan(0);
|
||||||
|
expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
|
||||||
|
expect(resolvedSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('in-flight probe captured before removal is discarded via tombstone', async () => {
|
||||||
|
let resolveProbe;
|
||||||
|
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||||
|
healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
|
||||||
|
resolveProbe = resolve;
|
||||||
|
}));
|
||||||
|
|
||||||
|
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
|
||||||
|
await pending;
|
||||||
|
|
||||||
|
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a rejected in-flight probe after removal does not re-create failure state', async () => {
|
||||||
|
let rejectProbe;
|
||||||
|
healthChecker.config.services.svc1 = { url: 'http://test.local' };
|
||||||
|
healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
|
||||||
|
rejectProbe = reject;
|
||||||
|
}));
|
||||||
|
|
||||||
|
const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
|
||||||
|
healthChecker.saveConfig = jest.fn();
|
||||||
|
healthChecker.removeService('svc1');
|
||||||
|
rejectProbe(new Error('late failure'));
|
||||||
|
await pending;
|
||||||
|
|
||||||
|
expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.currentStatus.has('svc1')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sweeps expired tombstones in cleanupHistory', () => {
|
||||||
|
healthChecker.removedGenerations.set('svc1', {
|
||||||
|
generation: 1,
|
||||||
|
removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
|
||||||
|
});
|
||||||
|
healthChecker.removedGenerations.set('svc2', {
|
||||||
|
generation: 2,
|
||||||
|
removedAt: Date.now() // fresh
|
||||||
|
});
|
||||||
|
|
||||||
|
healthChecker.cleanupHistory();
|
||||||
|
|
||||||
|
expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
|
||||||
|
expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('cleanupHistory', () => {
|
describe('cleanupHistory', () => {
|
||||||
it('removes entries older than retention period', () => {
|
it('removes entries older than retention period', () => {
|
||||||
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
|
const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
|
||||||
|
|||||||
@@ -26,6 +26,19 @@ jest.mock('dockerode', () => {
|
|||||||
function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk = true } = {}) {
|
function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk = true } = {}) {
|
||||||
process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
|
process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
|
||||||
|
|
||||||
|
// DC-087 — mirror src/app.js faithfully: the caddy check goes through
|
||||||
|
// fetchT (which injects the Origin header Caddy's enforce_origin allowlist
|
||||||
|
// requires), and is MOCKED so the suite is hermetic — no live request to a
|
||||||
|
// real Caddy admin on :2019. The previous raw-`fetch` mirror sent an
|
||||||
|
// Origin-less probe to the LIVE admin whenever the full suite ran on the
|
||||||
|
// prod host (adversarial cron every 30 min): 12 journal 403 lines per run,
|
||||||
|
// ~700/day of `client is not allowed to access from origin ''` noise,
|
||||||
|
// plus a false checks.caddy.ok=false in the mirrored readiness payload.
|
||||||
|
const fetchT = jest.spyOn(require('../src/utils/http'), 'fetchT')
|
||||||
|
.mockImplementation(async () => (caddyOk
|
||||||
|
? { ok: true, status: 200 }
|
||||||
|
: { ok: false, status: 403 }));
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
const config = {
|
const config = {
|
||||||
CONFIG_FILE: '/tmp/dc-test-config.json',
|
CONFIG_FILE: '/tmp/dc-test-config.json',
|
||||||
@@ -103,9 +116,13 @@ function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk
|
|||||||
allOk = false;
|
allOk = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DC-087 — mirror src/app.js exactly (fetchT, not raw fetch). fetchT is
|
||||||
|
// mocked at buildApp() scope, so this stays hermetic: no live probe to a
|
||||||
|
// real Caddy admin (the old raw-fetch mirror 403-spammed the prod journal
|
||||||
|
// every time the adversarial cron ran the full suite on this host).
|
||||||
try {
|
try {
|
||||||
const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
|
const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
|
||||||
const response = await fetch(`${caddyUrl}/config/apps/http/servers/srv0/listen`, { signal: AbortSignal.timeout(10000) });
|
const response = await fetchT(`${caddyUrl}/config/apps/http/servers/srv0/listen`, {}, 10000);
|
||||||
checks.caddy = { ok: response.ok, status: response.status };
|
checks.caddy = { ok: response.ok, status: response.status };
|
||||||
if (!response.ok) allOk = false;
|
if (!response.ok) allOk = false;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
@@ -33,9 +33,18 @@ jest.mock('dockerode', () => {
|
|||||||
|
|
||||||
// Mirror the canonical handler block from src/app.js — if this drifts from
|
// Mirror the canonical handler block from src/app.js — if this drifts from
|
||||||
// the real handler, these tests will start failing and force a sync.
|
// the real handler, these tests will start failing and force a sync.
|
||||||
function buildApp({ configOk = true, servicesOk = true, dockerOk = true } = {}) {
|
function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk = true } = {}) {
|
||||||
process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
|
process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
|
||||||
|
|
||||||
|
// DC-087 — mirror src/app.js: caddy check via fetchT (Origin-injecting),
|
||||||
|
// mocked here so the suite is hermetic. The old raw-fetch mirror probed the
|
||||||
|
// LIVE Caddy admin on :2019 whenever the full suite ran on the prod host
|
||||||
|
// (adversarial cron): Origin-less → 403 → 12 journal error lines per run.
|
||||||
|
const fetchT = jest.spyOn(require('../src/utils/http'), 'fetchT')
|
||||||
|
.mockImplementation(async () => (caddyOk
|
||||||
|
? { ok: true, status: 200 }
|
||||||
|
: { ok: false, status: 403 }));
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
const config = {
|
const config = {
|
||||||
CONFIG_FILE: '/tmp/dc-test-config.json',
|
CONFIG_FILE: '/tmp/dc-test-config.json',
|
||||||
@@ -108,8 +117,10 @@ function buildApp({ configOk = true, servicesOk = true, dockerOk = true } = {})
|
|||||||
allOk = false;
|
allOk = false;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
// DC-087 — mirror src/app.js exactly: fetchT (mocked above), not raw
|
||||||
|
// fetch. Hermetic: no live request to a real Caddy admin.
|
||||||
const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
|
const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
|
||||||
const response = await fetch(`${caddyUrl}/config/apps/http/servers/srv0/listen`, { signal: AbortSignal.timeout(10000) });
|
const response = await fetchT(`${caddyUrl}/config/apps/http/servers/srv0/listen`, {}, 10000);
|
||||||
checks.caddy = { ok: response.ok, status: response.status };
|
checks.caddy = { ok: response.ok, status: response.status };
|
||||||
if (!response.ok) allOk = false;
|
if (!response.ok) allOk = false;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
@@ -118,6 +118,67 @@ describe('Caddyfile + utils/http.js — Origin header construction (DC-051)', ()
|
|||||||
expect(offenders).toEqual([]);
|
expect(offenders).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('all :2019 call sites in TESTS use fetchT or a mocked fetchT (not raw fetch)', () => {
|
||||||
|
// DC-087 — the same rule, extended into __tests__. The api-code walk above
|
||||||
|
// skips __tests__, which let two mirrored health-handler test files keep a
|
||||||
|
// raw await-fetch caddy probe long after src/app.js moved to fetchT. On a
|
||||||
|
// host where the suite runs alongside a live Caddy admin (the prod box
|
||||||
|
// runs the full jest suite every 30 min via a cron adversarial check),
|
||||||
|
// that Origin-less raw fetch 403-spammed the Caddy journal (~700
|
||||||
|
// client-not-allowed error lines per day) while the tests still passed —
|
||||||
|
// checks.caddy.ok=false was silently accepted as sandbox noise. Mirrors
|
||||||
|
// MUST call fetchT (mocked at buildApp scope for hermeticity). A raw
|
||||||
|
// await-fetch at a Caddy-admin-URL call site in a test is an offender.
|
||||||
|
// NOTE: keep this comment free of backticks — stripComments pairs
|
||||||
|
// backtick spans across lines, and a stray pair shields real code from
|
||||||
|
// the comment stripper (this test self-flagged its first draft).
|
||||||
|
//
|
||||||
|
// Detection is deliberately FILE-LEVEL, not call-window: the historical
|
||||||
|
// drift kept the fetch call itself token-free (the URL came from a
|
||||||
|
// caddyUrl variable defined on a PREVIOUS line from CADDY_ADMIN_URL),
|
||||||
|
// so a call-window regex never fired. Any raw await-fetch in a file
|
||||||
|
// that also references the Caddy admin anywhere is an offender.
|
||||||
|
// Escape hatch for future tests that intentionally assert Origin-less
|
||||||
|
// 403 behavior against their own local listener: put the marker
|
||||||
|
// DC-087-ALLOW-RAW-FETCH in the file and it is skipped.
|
||||||
|
const testsRoot = path.join(__dirname);
|
||||||
|
const offenders = [];
|
||||||
|
const skipped = [];
|
||||||
|
function walk(dir) {
|
||||||
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||||
|
if (entry.name === 'node_modules') continue;
|
||||||
|
const p = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) walk(p);
|
||||||
|
else if (entry.name.endsWith('.js')) {
|
||||||
|
const rawText = fs.readFileSync(p, 'utf8');
|
||||||
|
// Escape hatch (checked on RAW text so a comment marker works —
|
||||||
|
// comments are stripped below): a file carrying the
|
||||||
|
// DC-087-ALLOW-RAW-FETCH marker declares it intentionally
|
||||||
|
// raw-fetches the Caddy admin (e.g. asserting Origin-less 403
|
||||||
|
// against its own local listener). The guard file itself is
|
||||||
|
// always scanned (never skipped) so the hatch can't be used to
|
||||||
|
// blind this very test.
|
||||||
|
if (p !== __filename && /DC-087-ALLOW-RAW-FETCH/.test(rawText)) {
|
||||||
|
skipped.push(p);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const text = stripComments(rawText);
|
||||||
|
const hasAdminToken = /:2019|adminUrl|admin_api_url|CADDY_ADMIN/.test(text);
|
||||||
|
const hasRawAwaitFetch = /await\s+fetch\(/.test(text);
|
||||||
|
if (hasAdminToken && hasRawAwaitFetch) {
|
||||||
|
offenders.push(`${p}: raw await-fetch in a file referencing the Caddy admin (mock fetchT instead; documented escape-hatch marker available for intentional 403 tests)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(testsRoot);
|
||||||
|
if (skipped.length) {
|
||||||
|
// Visibility for hatch use — shows up in jest output for reviewers.
|
||||||
|
console.info('[DC-087 guard] escape-hatch skipped:', skipped.join(', '));
|
||||||
|
}
|
||||||
|
expect(offenders).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
test('readiness handler in src/app.js probes the exact URL the watcher needs', () => {
|
test('readiness handler in src/app.js probes the exact URL the watcher needs', () => {
|
||||||
const raw = fs.readFileSync(
|
const raw = fs.readFileSync(
|
||||||
path.join(__dirname, '../src/app.js'),
|
path.join(__dirname, '../src/app.js'),
|
||||||
@@ -127,9 +188,9 @@ describe('Caddyfile + utils/http.js — Origin header construction (DC-051)', ()
|
|||||||
expect(raw).toMatch(/\/config\/apps\/http\/servers\/srv0\/listen/);
|
expect(raw).toMatch(/\/config\/apps\/http\/servers\/srv0\/listen/);
|
||||||
// Goes through fetchT, NOT bare fetch — that's how the Origin injection
|
// Goes through fetchT, NOT bare fetch — that's how the Origin injection
|
||||||
// takes effect. Look at the 800 chars BEFORE the probe URL on the same
|
// takes effect. Look at the 800 chars BEFORE the probe URL on the same
|
||||||
// line / call site — the call must be `fetchT(...)`, not `await fetch(...)`.
|
// line / call site — the call must be fetchT(...), never a raw await of
|
||||||
// (We look backward because the URL sits inside the call's argument list,
|
// the global fetch. (We look backward because the URL sits inside the
|
||||||
// so the call site comes before the URL token.)
|
// call's argument list, so the call site comes before the URL token.)
|
||||||
const idx = raw.indexOf('srv0/listen');
|
const idx = raw.indexOf('srv0/listen');
|
||||||
const around = raw.substr(Math.max(0, idx - 400), 800);
|
const around = raw.substr(Math.max(0, idx - 400), 800);
|
||||||
expect(around).toMatch(/fetchT\(/);
|
expect(around).toMatch(/fetchT\(/);
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ const platformPaths = require('../../platform-paths');
|
|||||||
const { createUserStore } = require('../../src/security/user-store');
|
const { createUserStore } = require('../../src/security/user-store');
|
||||||
const { createInviteStore } = require('../../src/security/invite-store');
|
const { createInviteStore } = require('../../src/security/invite-store');
|
||||||
const emailSender = require('../../src/auth/providers/email-sender');
|
const emailSender = require('../../src/auth/providers/email-sender');
|
||||||
|
const AuthProvider = require('../../src/auth/providers/base');
|
||||||
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
|
const { ValidationError, NotFoundError, ForbiddenError, PaymentRequiredError } = require('../../src/utilities/errors');
|
||||||
const { ok, successMessage } = require('../../src/utils/responses');
|
const { ok, successMessage } = require('../../src/utils/responses');
|
||||||
|
|
||||||
@@ -254,7 +255,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
|||||||
// server log on every unconfigured-install invite.
|
// server log on every unconfigured-install invite.
|
||||||
log.warn && log.warn('auth-invite-send',
|
log.warn && log.warn('auth-invite-send',
|
||||||
'invite send skipped: SMTP not configured (operator opted in)',
|
'invite send skipped: SMTP not configured (operator opted in)',
|
||||||
{ inviteId: issued.id, email: issued.email });
|
{ inviteId: issued.id, email: AuthProvider.maskEmail(issued.email) || '[unmaskable-email]' });
|
||||||
deliveredVia = 'failed';
|
deliveredVia = 'failed';
|
||||||
}
|
}
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
@@ -369,7 +370,7 @@ module.exports = function({ asyncHandler, errorResponse, log, session, dataDir }
|
|||||||
|
|
||||||
log.info && log.info('auth', 'invite accepted, user created', {
|
log.info && log.info('auth', 'invite accepted, user created', {
|
||||||
userId: userResult.user.id,
|
userId: userResult.user.id,
|
||||||
email: userResult.user.email,
|
email: AuthProvider.maskEmail(userResult.user.email) || '[unmaskable-email]',
|
||||||
role: userResult.user.role,
|
role: userResult.user.role,
|
||||||
inviteId: invite.id,
|
inviteId: invite.id,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -32,6 +32,10 @@ const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10
|
|||||||
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
|
||||||
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
|
||||||
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
|
||||||
|
// DC-088: how long a removal tombstone outlives the removal itself. Only needs
|
||||||
|
// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
|
||||||
|
// swept by cleanupHistory so removed services cannot accumulate map entries.
|
||||||
|
const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
|
||||||
|
|
||||||
// DC-086: hysteresis thresholds for badge display.
|
// DC-086: hysteresis thresholds for badge display.
|
||||||
// The raw probe result can flap on a single transient blip (Caddy reload,
|
// The raw probe result can flap on a single transient blip (Caddy reload,
|
||||||
@@ -72,6 +76,14 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
|
||||||
// Invalidate probe completions that race with removal/reconfiguration.
|
// Invalidate probe completions that race with removal/reconfiguration.
|
||||||
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
this.serviceGenerations = new Map(); // serviceId -> configuration generation
|
||||||
|
// DC-088: monotonically increasing sequence so generation numbers can never
|
||||||
|
// repeat across remove -> re-add cycles (prevents ABA on the stale check).
|
||||||
|
this.generationSeq = 0;
|
||||||
|
// DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
|
||||||
|
// serviceGenerations means the service is (re)configured; a tombstone with a
|
||||||
|
// HIGHER generation than the captured one marks the capture as stale. Entry
|
||||||
|
// is deleted when the service is removed, so the live map cannot leak.
|
||||||
|
this.removedGenerations = new Map();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -139,6 +151,21 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.cleanupHistory();
|
this.cleanupHistory();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DC-088: true when a probe's captured generation no longer matches the
|
||||||
|
* service's current configuration state. A live serviceGenerations entry
|
||||||
|
* must match exactly. With no live entry the service was never configured
|
||||||
|
* in this process (disk-loaded / direct callers) — stale only if a removal
|
||||||
|
* tombstone with a HIGHER generation exists.
|
||||||
|
*/
|
||||||
|
_isStaleCapture(serviceId, generation) {
|
||||||
|
if (this.serviceGenerations.has(serviceId)) {
|
||||||
|
return this.serviceGenerations.get(serviceId) !== generation;
|
||||||
|
}
|
||||||
|
const tomb = this.removedGenerations.get(serviceId);
|
||||||
|
return Boolean(tomb && tomb.generation > generation);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check a single service
|
* Check a single service
|
||||||
*/
|
*/
|
||||||
@@ -160,7 +187,7 @@ class HealthChecker extends EventEmitter {
|
|||||||
details: result.details
|
details: result.details
|
||||||
};
|
};
|
||||||
|
|
||||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
if (this._isStaleCapture(serviceId, generation)) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,9 +206,6 @@ class HealthChecker extends EventEmitter {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
const responseTime = Date.now() - startTime;
|
const responseTime = Date.now() - startTime;
|
||||||
|
|
||||||
// Increment failure count for backoff
|
|
||||||
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
|
||||||
|
|
||||||
const status = {
|
const status = {
|
||||||
serviceId,
|
serviceId,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
@@ -190,10 +214,14 @@ class HealthChecker extends EventEmitter {
|
|||||||
error: error.message
|
error: error.message
|
||||||
};
|
};
|
||||||
|
|
||||||
if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
|
if (this._isStaleCapture(serviceId, generation)) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Increment failure count for backoff — only after the result is known
|
||||||
|
// to be non-stale, so a removed service cannot re-create map entries.
|
||||||
|
this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
|
||||||
|
|
||||||
const previousStatus = this.currentStatus.get(serviceId);
|
const previousStatus = this.currentStatus.get(serviceId);
|
||||||
this.recordStatus(serviceId, status);
|
this.recordStatus(serviceId, status);
|
||||||
this.checkForIncidents(serviceId, status, config, previousStatus);
|
this.checkForIncidents(serviceId, status, config, previousStatus);
|
||||||
@@ -660,7 +688,13 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.config.services = {};
|
this.config.services = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
// DC-088: monotonic instance-wide sequence — a re-added service can never
|
||||||
|
// recycle a previous generation number, and any older in-flight capture is
|
||||||
|
// invalidated by definition.
|
||||||
|
this.generationSeq += 1;
|
||||||
|
this.serviceGenerations.set(serviceId, this.generationSeq);
|
||||||
|
// Re-configuration supersedes any prior removal tombstone.
|
||||||
|
this.removedGenerations.delete(serviceId);
|
||||||
this.config.services[serviceId] = {
|
this.config.services[serviceId] = {
|
||||||
enabled: config.enabled !== false,
|
enabled: config.enabled !== false,
|
||||||
name: config.name || serviceId,
|
name: config.name || serviceId,
|
||||||
@@ -683,12 +717,35 @@ class HealthChecker extends EventEmitter {
|
|||||||
* Remove service configuration
|
* Remove service configuration
|
||||||
*/
|
*/
|
||||||
removeService(serviceId) {
|
removeService(serviceId) {
|
||||||
this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
|
// DC-088: tombstone the captured generation instead of leaking an entry.
|
||||||
|
// The live map entry is deleted; an in-flight probe captured BEFORE this
|
||||||
|
// point sees no live entry but a higher tombstone generation, so it is
|
||||||
|
// discarded. configureService clears the tombstone on re-add.
|
||||||
|
this.generationSeq += 1;
|
||||||
|
this.serviceGenerations.delete(serviceId);
|
||||||
|
this.removedGenerations.set(serviceId, {
|
||||||
|
generation: this.generationSeq,
|
||||||
|
removedAt: Date.now()
|
||||||
|
});
|
||||||
if (this.config.services) {
|
if (this.config.services) {
|
||||||
delete this.config.services[serviceId];
|
delete this.config.services[serviceId];
|
||||||
this.saveConfig();
|
this.saveConfig();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DC-088: open incidents for a removed service must not linger forever.
|
||||||
|
// Close them through the same resolve path a recovery would, annotated so
|
||||||
|
// history shows why (dashboard renders resolved incidents green + duration).
|
||||||
|
for (const incident of this.incidents) {
|
||||||
|
if (incident.serviceId === serviceId && incident.status === 'open') {
|
||||||
|
incident.status = 'resolved';
|
||||||
|
incident.resolvedAt = new Date().toISOString();
|
||||||
|
incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
|
||||||
|
incident.resolvedBy = 'service-removed';
|
||||||
|
this.emit('incident-resolved', incident);
|
||||||
|
this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
this.currentStatus.delete(serviceId);
|
this.currentStatus.delete(serviceId);
|
||||||
this.displayedStatus.delete(serviceId);
|
this.displayedStatus.delete(serviceId);
|
||||||
this.consecutiveSinceChange.delete(serviceId);
|
this.consecutiveSinceChange.delete(serviceId);
|
||||||
@@ -714,6 +771,18 @@ class HealthChecker extends EventEmitter {
|
|||||||
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DC-088: sweep expired removal tombstones. After the TTL no probe that
|
||||||
|
// captured a pre-removal generation can still be in flight (timeout is
|
||||||
|
// bounded by performHealthCheck), so the tombstone has done its job.
|
||||||
|
if (this.removedGenerations.size > 0) {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [serviceId, tomb] of this.removedGenerations) {
|
||||||
|
if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
|
||||||
|
this.removedGenerations.delete(serviceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Vendored
+71
-71
File diff suppressed because one or more lines are too long
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
const CACHE = 'dashcaddy-shell-c25cea8485';
|
const CACHE = 'dashcaddy-shell-497e1f671c';
|
||||||
const PRECACHE = [
|
const PRECACHE = [
|
||||||
'/',
|
'/',
|
||||||
'/index.html',
|
'/index.html',
|
||||||
|
|||||||
Reference in New Issue
Block a user