Hermes
b1464d9b85
_saveState drops its private fixed-name .tmp + writeFileSync (no fsync) copy and delegates to src/utils/atomic-write.js atomicWriteJSON — same wx/fsync/rename/dir-fsync discipline as the six previously migrated stores (DC-099..DC-104). A crash can no longer tear caddy-upstreams.json (mute list + probe state): power loss through the old path could leave an empty/short state file and silently drop every mute; concurrent saves (60s probe loop vs setMuted) collided on the shared tmp name. Test mock extended with the fd-level fs API (openSync/writeSync/ fsyncSync/closeSync/unlinkSync + closedTmp stash) so the canonical path is exercised under the existing file-wide fs mock; fsState renamed mockFsState (jest.mock out-of-scope-variable hoist rule). New DC-105 pin: wx+fsync+rename required, no fixed .tmp, zero leftover tmp files, destination JSON complete with mute preserved. Judge: GLM-5.3 cold read, round-1 A/ship (deleg_5f57fcce). URN urn:ump:iuhgj3ajr5llshjasttsvelnptv6iqaek6xji5l3klcl72nseqdq Full suite: 121 suites / 2779 tests green.
679 lines
29 KiB
JavaScript
679 lines
29 KiB
JavaScript
/**
|
|
* Tests for caddy-upstream-watcher.
|
|
*
|
|
* Mock-driven: we stub fs (for /etc/caddy/sites scan + state file) and http/https
|
|
* (for the probe). Tests cover site parsing, probe happy/sad path, the 5-minute
|
|
* "dead" threshold, mute toggle, and incident integration with healthChecker.
|
|
*/
|
|
|
|
const path = require('path');
|
|
const Module = require('module');
|
|
|
|
// Mock fs with controllable behavior.
|
|
const mockFsState = {
|
|
files: {}, // path -> string content
|
|
exists: {}, // path -> bool
|
|
writeLog: [], // writeFileSync calls
|
|
fdMap: new Map(), // open fd -> { p, content } (DC-105 atomic-write path)
|
|
closedTmp: new Map(), // closed-but-not-yet-renamed tmp path -> content
|
|
nextFd: 0,
|
|
};
|
|
|
|
jest.mock('fs', () => {
|
|
const real = jest.requireActual('fs');
|
|
return {
|
|
...real,
|
|
existsSync: jest.fn((p) => mockFsState.exists[p] !== undefined ? mockFsState.exists[p] : (mockFsState.files[p] !== undefined)),
|
|
readFileSync: jest.fn((p) => {
|
|
if (mockFsState.files[p] === undefined) {
|
|
const e = new Error(`ENOENT: ${p}`);
|
|
e.code = 'ENOENT';
|
|
throw e;
|
|
}
|
|
return mockFsState.files[p];
|
|
}),
|
|
readdirSync: jest.fn((p) => Object.keys(mockFsState.files).filter(f => f.startsWith(p + '/')).map(f => f.substring(p.length + 1))),
|
|
writeFileSync: jest.fn((p, content) => {
|
|
mockFsState.writeLog.push({ p, content });
|
|
mockFsState.files[p] = content;
|
|
mockFsState.exists[p] = true;
|
|
}),
|
|
mkdirSync: jest.fn(),
|
|
// DC-105 canonical atomic-write path (atomic-write.js): openSync('wx') →
|
|
// writeSync → fsyncSync → closeSync → renameSync → dir fsync. Content
|
|
// accumulates per-fd, is stashed on close, and lands in files[] on rename.
|
|
openSync: jest.fn((p) => {
|
|
mockFsState.nextFd += 1;
|
|
mockFsState.fdMap.set(mockFsState.nextFd, { p, content: '' });
|
|
return mockFsState.nextFd;
|
|
}),
|
|
writeSync: jest.fn((fd, content) => {
|
|
const rec = mockFsState.fdMap.get(fd);
|
|
if (!rec) throw new Error(`EBADF: fd ${fd}`);
|
|
rec.content += content;
|
|
}),
|
|
fsyncSync: jest.fn(),
|
|
closeSync: jest.fn((fd) => {
|
|
const rec = mockFsState.fdMap.get(fd);
|
|
if (rec) {
|
|
mockFsState.closedTmp.set(rec.p, rec.content);
|
|
mockFsState.fdMap.delete(fd);
|
|
}
|
|
}),
|
|
renameSync: jest.fn((src, dst) => {
|
|
const content = mockFsState.closedTmp.has(src)
|
|
? mockFsState.closedTmp.get(src)
|
|
: mockFsState.files[src];
|
|
mockFsState.files[dst] = content;
|
|
mockFsState.exists[dst] = true;
|
|
mockFsState.closedTmp.delete(src);
|
|
delete mockFsState.files[src];
|
|
delete mockFsState.exists[src];
|
|
}),
|
|
unlinkSync: jest.fn()
|
|
};
|
|
});
|
|
|
|
// Mock http/https request to control probe responses.
|
|
const probeQueue = []; // each entry: { kind: 'ok'|'err'|'timeout'|'code', statusCode? }
|
|
jest.mock('http', () => ({
|
|
request: jest.fn((opts, cb) => {
|
|
const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
|
|
const handlers = {};
|
|
const res = {
|
|
statusCode: entry.statusCode || 200,
|
|
headers: { server: 'mock' },
|
|
resume: () => {},
|
|
on: (e, fn) => { handlers[e] = fn; }
|
|
};
|
|
const req = {
|
|
on: jest.fn((e, fn) => { handlers[e] = fn; }),
|
|
end: jest.fn(() => {
|
|
if (entry.kind === 'err') {
|
|
handlers.error && handlers.error(new Error(entry.message || 'connect ECONNREFUSED'));
|
|
return;
|
|
}
|
|
if (entry.kind === 'timeout') {
|
|
handlers.timeout && handlers.timeout();
|
|
return;
|
|
}
|
|
cb(res);
|
|
if (handlers.end) handlers.end();
|
|
}),
|
|
destroy: jest.fn()
|
|
};
|
|
return req;
|
|
})
|
|
}));
|
|
jest.mock('https', () => ({
|
|
request: jest.fn((opts, cb) => {
|
|
const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
|
|
const handlers = {};
|
|
const res = {
|
|
statusCode: entry.statusCode || 200,
|
|
headers: { server: 'mock-https' },
|
|
resume: () => {},
|
|
on: (e, fn) => { handlers[e] = fn; }
|
|
};
|
|
const req = {
|
|
on: jest.fn((e, fn) => { handlers[e] = fn; }),
|
|
end: jest.fn(() => {
|
|
if (entry.kind === 'err') {
|
|
handlers.error && handlers.error(new Error(entry.message || 'TLS error'));
|
|
return;
|
|
}
|
|
cb(res);
|
|
if (handlers.end) handlers.end();
|
|
}),
|
|
destroy: jest.fn()
|
|
};
|
|
return req;
|
|
})
|
|
}));
|
|
|
|
// Reset fs mock state between tests.
|
|
beforeEach(() => {
|
|
mockFsState.files = {};
|
|
mockFsState.exists = {};
|
|
mockFsState.writeLog = [];
|
|
mockFsState.fdMap = new Map();
|
|
mockFsState.closedTmp = new Map();
|
|
mockFsState.nextFd = 0;
|
|
probeQueue.length = 0;
|
|
jest.clearAllMocks();
|
|
jest.resetModules();
|
|
});
|
|
|
|
describe('CaddyUpstreamWatcher', () => {
|
|
const SITES = '/etc/caddy/sites';
|
|
const STATE = '/tmp/caddy-upstreams-test.json';
|
|
|
|
function seedSites(files) {
|
|
for (const [name, content] of Object.entries(files)) {
|
|
mockFsState.files[SITES + '/' + name] = content;
|
|
mockFsState.exists[SITES + '/' + name] = true;
|
|
}
|
|
}
|
|
|
|
function loadWatcher() {
|
|
process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
|
|
process.env.CADDY_SITES_DIR = SITES;
|
|
// Disable the singleton's auto-write so we can call _saveState manually.
|
|
const mod = require('../src/monitoring/caddy-upstream-watcher');
|
|
return { mod, w: mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod };
|
|
}
|
|
|
|
test('parses reverse_proxy directives from /etc/caddy/sites/*', async () => {
|
|
seedSites({
|
|
'arch.sami': `arch.sami {\n reverse_proxy 100.120.159.34:5000 { health_uri /api/stats }\n}`,
|
|
'appt.sami': `appt.sami {\n reverse_proxy http://100.81.59.99:5232\n}`,
|
|
'zap.sami': `zap.sami {\n reverse_proxy 10.0.0.5:8080\n reverse_proxy 10.0.0.6:8080 # multiple upstreams in same site block\n}`
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const snap = w.snapshot();
|
|
const hosts = snap.upstreams.map(u => u.host).sort();
|
|
expect(hosts).toEqual(['10.0.0.5:8080', '10.0.0.6:8080', '100.120.159.34:5000', '100.81.59.99:5232']);
|
|
expect(snap.upstreams.find(u => u.host === '100.120.159.34:5000').site).toBe('arch.sami');
|
|
expect(snap.upstreams.find(u => u.host === '100.81.59.99:5232').site).toBe('appt.sami');
|
|
});
|
|
|
|
test('ignores non-site files and unparseable entries', async () => {
|
|
seedSites({
|
|
'README.md': '# documentation\nreverse_proxy 1.2.3.4:9999\n', // not a site file
|
|
'garbage.sami': 'not a caddyfile\n', // no reverse_proxy
|
|
'good.sami': 'good.sami {\n reverse_proxy 1.2.3.4:9999\n}\n'
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const hosts = w.snapshot().upstreams.map(u => u.host);
|
|
expect(hosts).toEqual(['1.2.3.4:9999']);
|
|
});
|
|
|
|
test('handles real prod-style filenames: zap.sami-ahmed.net, samitest.space, blocks.cryptographic-triangles.org', async () => {
|
|
// These are the actual file names in production /etc/caddy/sites/ —
|
|
// extension is .net / .space / .org, NOT .sami/.caddy/.conf. The old
|
|
// file-extension filter would skip them silently.
|
|
seedSites({
|
|
'zap.sami-ahmed.net': 'zap.sami-ahmed.net {\n reverse_proxy localhost:8088\n}\n',
|
|
'samitest.space': 'samitest.space {\n reverse_proxy 100.120.159.34:8080 {}\n}\n',
|
|
'blocks.cryptographic-triangles.org': 'blocks.cryptographic-triangles.org {\n\treverse_proxy localhost:3052 {}\n}\n'
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const snap = w.snapshot();
|
|
const byHost = Object.fromEntries(snap.upstreams.map(u => [u.host, u.site]));
|
|
expect(byHost['localhost:8088']).toBe('zap.sami-ahmed.net');
|
|
expect(byHost['100.120.159.34:8080']).toBe('samitest.space');
|
|
expect(byHost['localhost:3052']).toBe('blocks.cryptographic-triangles.org');
|
|
});
|
|
|
|
test('drops upstreams that disappear from the sites dir', async () => {
|
|
seedSites({
|
|
'arch.sami': 'arch.sami {\n reverse_proxy 1.1.1.1:5000\n}\n'
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
expect(w.upstreams.size).toBe(1);
|
|
mockFsState.files = {}; // wipe
|
|
mockFsState.exists = {};
|
|
await w.scanSites();
|
|
expect(w.upstreams.size).toBe(0);
|
|
});
|
|
|
|
test('healthy probe updates state and does not open an incident', async () => {
|
|
seedSites({ 'good.sami': 'good.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
await w._probeOne(w.upstreams.values().next().value);
|
|
const snap = w.snapshot();
|
|
expect(snap.upstreams[0].status).toBe('up');
|
|
expect(snap.upstreams[0].lastSuccessAt).toBeTruthy();
|
|
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test('auth-walled 4xx counts as healthy (proves the upstream answered)', async () => {
|
|
seedSites({ 'auth.sami': 'auth.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 401 });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
await w._probeOne(w.upstreams.values().next().value);
|
|
expect(w.snapshot().upstreams[0].status).toBe('up');
|
|
});
|
|
|
|
test('first failure flips status to down but does NOT open an incident (under 5min)', async () => {
|
|
seedSites({ 'bad.sami': 'bad.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED' });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.values().next().value;
|
|
u.lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 30s ago it was healthy
|
|
await w._probeOne(u);
|
|
const snap = w.snapshot();
|
|
expect(snap.upstreams[0].status).toBe('down');
|
|
expect(snap.upstreams[0].failingForMs).toBeLessThan(5 * 60 * 1000);
|
|
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test('after 5 minutes of consecutive failures an incident is opened', async () => {
|
|
seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
|
|
const { w } = loadWatcher();
|
|
const incidents = [];
|
|
const fakeHealthChecker = {
|
|
createIncident: jest.fn((serviceId, type, message, status) => {
|
|
incidents.push({ serviceId, type, message, status });
|
|
}),
|
|
incidents: []
|
|
};
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.values().next().value;
|
|
// Simulate lastSuccessAt being 6 minutes ago so failingForMs exceeds DEAD_AFTER_MS.
|
|
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
|
|
await w._probeOne(u);
|
|
expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
|
|
expect(fakeHealthChecker.createIncident.mock.calls[0][1]).toBe('caddy-upstream-dead');
|
|
expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
|
|
});
|
|
|
|
test('does not duplicate incidents for the same upstream', async () => {
|
|
seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
// Queue up 3 errors so each probe fails.
|
|
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
|
|
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
|
|
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = {
|
|
createIncident: jest.fn(),
|
|
incidents: []
|
|
};
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.values().next().value;
|
|
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
|
|
await w._probeOne(u);
|
|
await w._probeOne(u);
|
|
await w._probeOne(u);
|
|
expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
test('recovery resolves the open incident after RESOLVED_AFTER_MS', async () => {
|
|
seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
probeQueue.push({ kind: 'err', message: 'i/o timeout' }); // trip dead
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 }); // recovery
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = {
|
|
createIncident: jest.fn(),
|
|
resolveIncident: jest.fn(),
|
|
incidents: []
|
|
};
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.values().next().value;
|
|
// Trip the dead state
|
|
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
|
|
await w._probeOne(u);
|
|
expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
|
|
// Simulate a recovery — lastFailureAt is 2 min ago, now healthy
|
|
u.lastFailureAt = new Date(Date.now() - 2 * 60 * 1000).toISOString();
|
|
await w._probeOne(u);
|
|
expect(fakeHealthChecker.resolveIncident).toHaveBeenCalledTimes(1);
|
|
expect(w.openIncidents.has('1.1.1.1:80')).toBe(false);
|
|
});
|
|
|
|
test('mute suppresses probing and hides upstream in snapshot status', async () => {
|
|
seedSites({ 'noisy.sami': 'noisy.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
w.setMuted('1.1.1.1:80', true);
|
|
expect(w.isMuted('1.1.1.1:80')).toBe(true);
|
|
const snap = w.snapshot();
|
|
expect(snap.upstreams[0].status).toBe('muted');
|
|
expect(snap.upstreams[0].muted).toBe(true);
|
|
// probe tick should skip muted
|
|
await w._tick();
|
|
// lastCheckedAt should NOT have advanced because no probe was issued
|
|
expect(snap.upstreams[0].lastCheckedAt).toBeNull();
|
|
});
|
|
|
|
test('unmute resets failure counters so a recently-recovered upstream is not immediately re-incidented', async () => {
|
|
seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const u = w.upstreams.values().next().value;
|
|
u.consecutiveFailures = 42;
|
|
u.lastError = 'old failure';
|
|
u.lastFailureAt = new Date().toISOString();
|
|
u.status = 'down';
|
|
w.setMuted('1.1.1.1:80', true);
|
|
w.setMuted('1.1.1.1:80', false);
|
|
expect(u.consecutiveFailures).toBe(0);
|
|
expect(u.status).toBe('unknown');
|
|
expect(u.lastError).toBeNull();
|
|
});
|
|
|
|
test('snapshot sorts dead > down > muted > up > unknown', async () => {
|
|
seedSites({
|
|
'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
|
|
'b.sami': 'b.sami { reverse_proxy 2.2.2.2:80 }\n',
|
|
'c.sami': 'c.sami { reverse_proxy 3.3.3.3:80 }\n',
|
|
'd.sami': 'd.sami { reverse_proxy 4.4.4.4:80 }\n',
|
|
'e.sami': 'e.sami { reverse_proxy 5.5.5.5:80 }\n'
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const all = Array.from(w.upstreams.values());
|
|
// 1.1.1.1:80 -> up (just succeeded)
|
|
all.find(u => u.host === '1.1.1.1:80').status = 'up';
|
|
all.find(u => u.host === '1.1.1.1:80').lastSuccessAt = new Date().toISOString();
|
|
// 2.2.2.2:80 -> down (recent — last success 30s ago)
|
|
all.find(u => u.host === '2.2.2.2:80').status = 'down';
|
|
all.find(u => u.host === '2.2.2.2:80').lastSuccessAt = new Date(Date.now() - 30000).toISOString();
|
|
// 3.3.3.3:80 -> muted
|
|
w.muted.add('3.3.3.3:80');
|
|
// 4.4.4.4:80 -> dead (last success 7min ago, never recovered)
|
|
const dead = all.find(u => u.host === '4.4.4.4:80');
|
|
dead.status = 'down';
|
|
dead.lastSuccessAt = new Date(Date.now() - 7 * 60 * 1000).toISOString();
|
|
// 5.5.5.5:80 -> unknown (no probes yet)
|
|
const snap = w.snapshot();
|
|
const order = snap.upstreams.map(u => u.host);
|
|
// Expected: dead first, then down, then muted, then up, then unknown
|
|
expect(order).toEqual(['4.4.4.4:80', '2.2.2.2:80', '3.3.3.3:80', '1.1.1.1:80', '5.5.5.5:80']);
|
|
});
|
|
|
|
test('persists muted list to state file', async () => {
|
|
seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
w.setMuted('1.1.1.1:80', true);
|
|
// DC-105: _saveState delegates to atomicWriteJSON — content lands via
|
|
// openSync('wx')+writeSync+rename, not writeFileSync to a fixed .tmp.
|
|
// The renamed destination must carry the muted host.
|
|
expect(mockFsState.exists[STATE]).toBe(true);
|
|
const data = JSON.parse(mockFsState.files[STATE]);
|
|
expect(data.muted).toContain('1.1.1.1:80');
|
|
// And the legacy fixed-name tmp path must NOT have been used.
|
|
expect(mockFsState.writeLog.filter(w => w.p === STATE + '.tmp').length).toBe(0);
|
|
});
|
|
|
|
// ---- DC-105: state file goes through the canonical atomic-write util ------
|
|
|
|
test('DC-105: _saveState uses atomicWriteJSON (wx tmp + fsync + rename, no fixed .tmp)', async () => {
|
|
seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
w.setMuted('1.1.1.1:80', true);
|
|
|
|
const fs = require('fs');
|
|
// The canonical writer must have been used: open with 'wx' (exclusive
|
|
// create), fsync before close, then rename onto the destination.
|
|
expect(fs.openSync).toHaveBeenCalled();
|
|
expect(fs.fsyncSync).toHaveBeenCalled();
|
|
expect(fs.closeSync).toHaveBeenCalled();
|
|
const renames = fs.renameSync.mock.calls.filter(c => c[1] === STATE);
|
|
expect(renames.length).toBeGreaterThan(0);
|
|
// Tmp names are hidden dotfiles in the same dir with pid+counter — the
|
|
// old fixed `STATE + '.tmp'` collision window between concurrent saves
|
|
// (probe loop vs setMuted) is gone.
|
|
for (const [src] of renames) {
|
|
expect(src).toMatch(/[\\/].caddy-upstreams-test[.]json[.]tmp-/);
|
|
expect(src).not.toBe(STATE + '.tmp');
|
|
}
|
|
// No leftover tmp files after a successful save.
|
|
const leftovers = Object.keys(mockFsState.files)
|
|
.filter(p => p.includes('.tmp-'));
|
|
expect(leftovers).toEqual([]);
|
|
// Destination holds complete, parseable JSON with the mute.
|
|
const data = JSON.parse(mockFsState.files[STATE]);
|
|
expect(data.muted).toContain('1.1.1.1:80');
|
|
expect(data.upstreams['1.1.1.1:80'].site).toBe('a.sami');
|
|
});
|
|
|
|
test('reload from state file restores muted list', async () => {
|
|
// Pre-seed a state file with a muted host
|
|
mockFsState.files[STATE] = JSON.stringify({
|
|
muted: ['99.99.99.99:80'],
|
|
upstreams: { '99.99.99.99:80': { ip: '99.99.99.99', port: '80', site: 'old.sami', siteFile: 'old.sami' } }
|
|
});
|
|
mockFsState.exists[STATE] = true;
|
|
// And the matching site file
|
|
seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' });
|
|
process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
|
|
process.env.CADDY_SITES_DIR = SITES;
|
|
const mod = require('../src/monitoring/caddy-upstream-watcher');
|
|
const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod;
|
|
expect(w.isMuted('99.99.99.99:80')).toBe(true);
|
|
});
|
|
|
|
// ---- Loopback → host-gateway remap (live bug 2026-08-18) -----------------
|
|
// The watcher runs inside the container; Caddyfile `localhost:PORT` means
|
|
// the HOST's loopback. Probing the container's own loopback gave 278
|
|
// phantom failures per healthy host-side upstream.
|
|
|
|
test('loopback upstream probe is remapped to host.docker.internal (not container loopback)', async () => {
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const u = w.upstreams.get('localhost:8088');
|
|
expect(u).toBeTruthy();
|
|
const http = require('http');
|
|
await w._probeOne(u);
|
|
// The probe request must have gone to host.docker.internal, keeping the port.
|
|
const call = http.request.mock.calls.find(c => c[0].hostname === 'host.docker.internal');
|
|
expect(call).toBeTruthy();
|
|
expect(call[0].port).toBe('8088');
|
|
// Display key is unchanged.
|
|
expect(w.snapshot().upstreams[0].host).toBe('localhost:8088');
|
|
expect(w.snapshot().upstreams[0].status).toBe('up');
|
|
});
|
|
|
|
test('127.0.0.1 and 127.x addresses are also remapped', async () => {
|
|
seedSites({
|
|
'a.sami': 'a.sami { reverse_proxy 127.0.0.1:8765 }\n',
|
|
'b.sami': 'b.sami { reverse_proxy 127.0.0.53:9000 }\n'
|
|
});
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const http = require('http');
|
|
for (const u of w.upstreams.values()) await w._probeOne(u);
|
|
const hostnames = http.request.mock.calls.map(c => c[0].hostname);
|
|
expect(hostnames).toEqual(['host.docker.internal', 'host.docker.internal']);
|
|
});
|
|
|
|
test('non-loopback upstreams are probed at their literal address (no remap)', async () => {
|
|
seedSites({ 'arch.sami': 'arch.sami { reverse_proxy 100.120.159.34:5000 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const http = require('http');
|
|
await w._probeOne(w.upstreams.get('100.120.159.34:5000'));
|
|
const hostnames = http.request.mock.calls.map(c => c[0].hostname);
|
|
expect(hostnames).toEqual(['100.120.159.34']);
|
|
});
|
|
|
|
test('failed host-gateway probe marks loopback upstream unverifiable — no failures, no incident', async () => {
|
|
// Services bound to 127.0.0.1 on the host refuse bridge-IP connections;
|
|
// from inside the container that is indistinguishable from "dead", and
|
|
// Caddy (on the host) still routes fine — so it must NOT count as down.
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED 172.17.0.1:8088' });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] };
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.get('localhost:8088');
|
|
u.lastSuccessAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); // long "failing" history
|
|
await w._probeOne(u);
|
|
const snap = w.snapshot().upstreams[0];
|
|
expect(snap.status).toBe('unverifiable');
|
|
expect(snap.consecutiveFailures).toBe(0);
|
|
expect(snap.dead).toBe(false);
|
|
expect(snap.failingForMs).toBe(0);
|
|
expect(snap.lastError).toMatch(/not verifiable from container/);
|
|
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test('unverifiable sorts between muted and up in the snapshot', async () => {
|
|
seedSites({
|
|
'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
|
|
'b.sami': 'b.sami { reverse_proxy localhost:9876 }\n',
|
|
'c.sami': 'c.sami { reverse_proxy 2.2.2.2:80 }\n'
|
|
});
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const all = Array.from(w.upstreams.values());
|
|
all.find(u => u.host === '1.1.1.1:80').status = 'up';
|
|
all.find(u => u.host === 'localhost:9876').status = 'unverifiable';
|
|
w.muted.add('2.2.2.2:80');
|
|
const order = w.snapshot().upstreams.map(u => u.host);
|
|
expect(order).toEqual(['2.2.2.2:80', 'localhost:9876', '1.1.1.1:80']);
|
|
});
|
|
|
|
// ---- verifiedViaBridge (DC-053 follow-up item 2b-a) ----------------------
|
|
// A loopback upstream whose PRIOR probe succeeded via host-gateway proves
|
|
// the bridge CAN reach the host. If a later probe then fails, that is
|
|
// near-conclusive evidence the upstream itself went dead — not that
|
|
// bridge connectivity broke. Restore dead-detection for that subset.
|
|
|
|
test('successful host-gateway probe sets verifiedViaBridge=true on loopback upstream', async () => {
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const u = w.upstreams.get('localhost:8088');
|
|
expect(u.verifiedViaBridge).toBeFalsy();
|
|
await w._probeOne(u);
|
|
expect(u.verifiedViaBridge).toBe(true);
|
|
expect(u.status).toBe('up');
|
|
expect(w.snapshot().upstreams[0].verifiedViaBridge).toBe(true);
|
|
});
|
|
|
|
test('loopback upstream with verifiedViaBridge fails -> counts as down (not unverifiable)', async () => {
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
// First probe succeeds (sets verifiedViaBridge), second probe fails.
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED 172.17.0.1:8088' });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] };
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.get('localhost:8088');
|
|
await w._probeOne(u);
|
|
expect(u.verifiedViaBridge).toBe(true);
|
|
expect(u.status).toBe('up');
|
|
await w._probeOne(u);
|
|
expect(u.status).toBe('down');
|
|
expect(u.consecutiveFailures).toBe(1);
|
|
expect(u.lastError).toMatch(/ECONNREFUSED/);
|
|
// Snapshot also reflects verifiedViaBridge so dashboard can label it.
|
|
const snap = w.snapshot().upstreams[0];
|
|
expect(snap.verifiedViaBridge).toBe(true);
|
|
// No incident yet — needs DEAD_AFTER_MS of continuous failure.
|
|
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test('loopback upstream with verifiedViaBridge eventually opens a dead incident', async () => {
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 }); // probe 1: success -> verifiedViaBridge
|
|
probeQueue.push({ kind: 'err', message: 'down' });
|
|
const { w } = loadWatcher();
|
|
const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] };
|
|
w.healthChecker = fakeHealthChecker;
|
|
await w.scanSites();
|
|
const u = w.upstreams.get('localhost:8088');
|
|
await w._probeOne(u);
|
|
// Pre-set lastSuccessAt to DEAD_AFTER_MS ago so the second failure
|
|
// immediately crosses the 5-minute threshold.
|
|
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
|
|
await w._probeOne(u);
|
|
expect(fakeHealthChecker.createIncident).toHaveBeenCalledWith(
|
|
'localhost:8088',
|
|
'caddy-upstream-dead',
|
|
expect.stringMatching(/unreachable for 6m/),
|
|
expect.objectContaining({ status: 'down', serviceId: 'localhost:8088' })
|
|
);
|
|
});
|
|
|
|
// ---- IN_CONTAINER=false kill-switch (DC-053 follow-up item 2b-b) --------
|
|
// When the API runs bare-metal (or in a sidecar next to Caddy), the
|
|
// loopback host IS the host — no bridge. Probing loopback verbatim
|
|
// gives real, conclusive evidence.
|
|
|
|
test('IN_CONTAINER=false disables host-gateway remap (probes loopback verbatim)', async () => {
|
|
process.env.IN_CONTAINER = 'false';
|
|
try {
|
|
seedSites({
|
|
'a.sami': 'a.sami { reverse_proxy localhost:8088 }\n',
|
|
'b.sami': 'b.sami { reverse_proxy 127.0.0.1:9000 }\n',
|
|
'c.sami': 'c.sami { reverse_proxy 1.2.3.4:80 }\n' // non-loopback, should still go literal
|
|
});
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
// Force module reload so the new IN_CONTAINER is picked up at require time.
|
|
jest.resetModules();
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const http = require('http');
|
|
for (const u of w.upstreams.values()) await w._probeOne(u);
|
|
const hostnames = http.request.mock.calls.map(c => c[0].hostname);
|
|
// All three go to their literal addresses — no host.docker.internal.
|
|
expect(hostnames).toEqual(['localhost', '127.0.0.1', '1.2.3.4']);
|
|
// And no upstream is marked verifiedViaBridge (the loopback-success
|
|
// gate only matters in the bridge case).
|
|
for (const u of w.upstreams.values()) {
|
|
expect(u.verifiedViaBridge).toBeFalsy();
|
|
}
|
|
} finally {
|
|
delete process.env.IN_CONTAINER;
|
|
}
|
|
});
|
|
|
|
test('IN_CONTAINER unset (default) still uses host-gateway remap', async () => {
|
|
delete process.env.IN_CONTAINER;
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
|
jest.resetModules();
|
|
const { w } = loadWatcher();
|
|
await w.scanSites();
|
|
const http = require('http');
|
|
await w._probeOne(w.upstreams.get('localhost:8088'));
|
|
const call = http.request.mock.calls.find(c => c[0].hostname === 'host.docker.internal');
|
|
expect(call).toBeTruthy();
|
|
});
|
|
|
|
// ---- verifiedViaBridge persistence (B-grade polish) -----------------------
|
|
// GLM judge LOW: don't re-prove bridge connectivity across container
|
|
// restarts. A previously-positive observation is still good evidence.
|
|
|
|
test('verifiedViaBridge survives a save -> reload cycle (state.json round-trip)', async () => {
|
|
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
|
probeQueue.push({ kind: 'ok', statusCode: 200 }); // probe succeeds -> verifiedViaBridge=true
|
|
const { w: w1 } = loadWatcher();
|
|
await w1.scanSites();
|
|
const u = w1.upstreams.get('localhost:8088');
|
|
await w1._probeOne(u);
|
|
expect(u.verifiedViaBridge).toBe(true);
|
|
// Force a save.
|
|
w1._saveState();
|
|
// Reload from the same file via a fresh watcher instance.
|
|
jest.resetModules();
|
|
const { w: w2 } = loadWatcher();
|
|
await w2.scanSites();
|
|
const restored = w2.upstreams.get('localhost:8088');
|
|
expect(restored).toBeTruthy();
|
|
expect(restored.verifiedViaBridge).toBe(true);
|
|
// The snapshot field carries it through too.
|
|
expect(w2.snapshot().upstreams[0].verifiedViaBridge).toBe(true);
|
|
});
|
|
}); |