/** * Tests for caddy-upstream-watcher. * * Mock-driven: we stub fs (for /etc/caddy/sites scan + state file) and http/https * (for the probe). Tests cover site parsing, probe happy/sad path, the 5-minute * "dead" threshold, mute toggle, and incident integration with healthChecker. */ const path = require('path'); const Module = require('module'); // Mock fs with controllable behavior. const fsState = { files: {}, // path -> string content exists: {}, // path -> bool writeLog: [], // writes }; jest.mock('fs', () => { const real = jest.requireActual('fs'); return { ...real, existsSync: jest.fn((p) => fsState.exists[p] !== undefined ? fsState.exists[p] : (fsState.files[p] !== undefined)), readFileSync: jest.fn((p) => { if (fsState.files[p] === undefined) { const e = new Error(`ENOENT: ${p}`); e.code = 'ENOENT'; throw e; } return fsState.files[p]; }), readdirSync: jest.fn((p) => Object.keys(fsState.files).filter(f => f.startsWith(p + '/')).map(f => f.substring(p.length + 1))), writeFileSync: jest.fn((p, content) => { fsState.writeLog.push({ p, content }); fsState.files[p] = content; fsState.exists[p] = true; }), mkdirSync: jest.fn(), renameSync: jest.fn((src, dst) => { fsState.files[dst] = fsState.files[src]; fsState.exists[dst] = true; delete fsState.files[src]; delete fsState.exists[src]; }) }; }); // Mock http/https request to control probe responses. const probeQueue = []; // each entry: { kind: 'ok'|'err'|'timeout'|'code', statusCode? } jest.mock('http', () => ({ request: jest.fn((opts, cb) => { const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 }; const handlers = {}; const res = { statusCode: entry.statusCode || 200, headers: { server: 'mock' }, resume: () => {}, on: (e, fn) => { handlers[e] = fn; } }; const req = { on: jest.fn((e, fn) => { handlers[e] = fn; }), end: jest.fn(() => { if (entry.kind === 'err') { handlers.error && handlers.error(new Error(entry.message || 'connect ECONNREFUSED')); return; } if (entry.kind === 'timeout') { handlers.timeout && handlers.timeout(); return; } cb(res); if (handlers.end) handlers.end(); }), destroy: jest.fn() }; return req; }) })); jest.mock('https', () => ({ request: jest.fn((opts, cb) => { const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 }; const handlers = {}; const res = { statusCode: entry.statusCode || 200, headers: { server: 'mock-https' }, resume: () => {}, on: (e, fn) => { handlers[e] = fn; } }; const req = { on: jest.fn((e, fn) => { handlers[e] = fn; }), end: jest.fn(() => { if (entry.kind === 'err') { handlers.error && handlers.error(new Error(entry.message || 'TLS error')); return; } cb(res); if (handlers.end) handlers.end(); }), destroy: jest.fn() }; return req; }) })); // Reset fs mock state between tests. beforeEach(() => { fsState.files = {}; fsState.exists = {}; fsState.writeLog = []; probeQueue.length = 0; jest.clearAllMocks(); jest.resetModules(); }); describe('CaddyUpstreamWatcher', () => { const SITES = '/etc/caddy/sites'; const STATE = '/tmp/caddy-upstreams-test.json'; function seedSites(files) { for (const [name, content] of Object.entries(files)) { fsState.files[SITES + '/' + name] = content; fsState.exists[SITES + '/' + name] = true; } } function loadWatcher() { process.env.CADDY_UPSTREAMS_STATE_FILE = STATE; process.env.CADDY_SITES_DIR = SITES; // Disable the singleton's auto-write so we can call _saveState manually. const mod = require('../src/monitoring/caddy-upstream-watcher'); return { mod, w: mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod }; } test('parses reverse_proxy directives from /etc/caddy/sites/*', async () => { seedSites({ 'arch.sami': `arch.sami {\n reverse_proxy 100.120.159.34:5000 { health_uri /api/stats }\n}`, 'appt.sami': `appt.sami {\n reverse_proxy http://100.81.59.99:5232\n}`, 'zap.sami': `zap.sami {\n reverse_proxy 10.0.0.5:8080\n reverse_proxy 10.0.0.6:8080 # multiple upstreams in same site block\n}` }); const { w } = loadWatcher(); await w.scanSites(); const snap = w.snapshot(); const hosts = snap.upstreams.map(u => u.host).sort(); expect(hosts).toEqual(['10.0.0.5:8080', '10.0.0.6:8080', '100.120.159.34:5000', '100.81.59.99:5232']); expect(snap.upstreams.find(u => u.host === '100.120.159.34:5000').site).toBe('arch.sami'); expect(snap.upstreams.find(u => u.host === '100.81.59.99:5232').site).toBe('appt.sami'); }); test('ignores non-site files and unparseable entries', async () => { seedSites({ 'README.md': '# documentation\nreverse_proxy 1.2.3.4:9999\n', // not a site file 'garbage.sami': 'not a caddyfile\n', // no reverse_proxy 'good.sami': 'good.sami {\n reverse_proxy 1.2.3.4:9999\n}\n' }); const { w } = loadWatcher(); await w.scanSites(); const hosts = w.snapshot().upstreams.map(u => u.host); expect(hosts).toEqual(['1.2.3.4:9999']); }); test('handles real prod-style filenames: zap.sami-ahmed.net, samitest.space, blocks.cryptographic-triangles.org', async () => { // These are the actual file names in production /etc/caddy/sites/ — // extension is .net / .space / .org, NOT .sami/.caddy/.conf. The old // file-extension filter would skip them silently. seedSites({ 'zap.sami-ahmed.net': 'zap.sami-ahmed.net {\n reverse_proxy localhost:8088\n}\n', 'samitest.space': 'samitest.space {\n reverse_proxy 100.120.159.34:8080 {}\n}\n', 'blocks.cryptographic-triangles.org': 'blocks.cryptographic-triangles.org {\n\treverse_proxy localhost:3052 {}\n}\n' }); const { w } = loadWatcher(); await w.scanSites(); const snap = w.snapshot(); const byHost = Object.fromEntries(snap.upstreams.map(u => [u.host, u.site])); expect(byHost['localhost:8088']).toBe('zap.sami-ahmed.net'); expect(byHost['100.120.159.34:8080']).toBe('samitest.space'); expect(byHost['localhost:3052']).toBe('blocks.cryptographic-triangles.org'); }); test('drops upstreams that disappear from the sites dir', async () => { seedSites({ 'arch.sami': 'arch.sami {\n reverse_proxy 1.1.1.1:5000\n}\n' }); const { w } = loadWatcher(); await w.scanSites(); expect(w.upstreams.size).toBe(1); fsState.files = {}; // wipe fsState.exists = {}; await w.scanSites(); expect(w.upstreams.size).toBe(0); }); test('healthy probe updates state and does not open an incident', async () => { seedSites({ 'good.sami': 'good.sami { reverse_proxy 1.1.1.1:80 }\n' }); probeQueue.push({ kind: 'ok', statusCode: 200 }); const { w } = loadWatcher(); const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] }; w.healthChecker = fakeHealthChecker; await w.scanSites(); await w._probeOne(w.upstreams.values().next().value); const snap = w.snapshot(); expect(snap.upstreams[0].status).toBe('up'); expect(snap.upstreams[0].lastSuccessAt).toBeTruthy(); expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled(); }); test('auth-walled 4xx counts as healthy (proves the upstream answered)', async () => { seedSites({ 'auth.sami': 'auth.sami { reverse_proxy 1.1.1.1:80 }\n' }); probeQueue.push({ kind: 'ok', statusCode: 401 }); const { w } = loadWatcher(); await w.scanSites(); await w._probeOne(w.upstreams.values().next().value); expect(w.snapshot().upstreams[0].status).toBe('up'); }); test('first failure flips status to down but does NOT open an incident (under 5min)', async () => { seedSites({ 'bad.sami': 'bad.sami { reverse_proxy 1.1.1.1:80 }\n' }); probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED' }); const { w } = loadWatcher(); const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] }; w.healthChecker = fakeHealthChecker; await w.scanSites(); const u = w.upstreams.values().next().value; u.lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 30s ago it was healthy await w._probeOne(u); const snap = w.snapshot(); expect(snap.upstreams[0].status).toBe('down'); expect(snap.upstreams[0].failingForMs).toBeLessThan(5 * 60 * 1000); expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled(); }); test('after 5 minutes of consecutive failures an incident is opened', async () => { seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' }); probeQueue.push({ kind: 'err', message: 'i/o timeout' }); const { w } = loadWatcher(); const incidents = []; const fakeHealthChecker = { createIncident: jest.fn((serviceId, type, message, status) => { incidents.push({ serviceId, type, message, status }); }), incidents: [] }; w.healthChecker = fakeHealthChecker; await w.scanSites(); const u = w.upstreams.values().next().value; // Simulate lastSuccessAt being 6 minutes ago so failingForMs exceeds DEAD_AFTER_MS. u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); await w._probeOne(u); expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1); expect(fakeHealthChecker.createIncident.mock.calls[0][1]).toBe('caddy-upstream-dead'); expect(w.openIncidents.has('1.1.1.1:80')).toBe(true); }); test('does not duplicate incidents for the same upstream', async () => { seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' }); // Queue up 3 errors so each probe fails. probeQueue.push({ kind: 'err', message: 'i/o timeout' }); probeQueue.push({ kind: 'err', message: 'i/o timeout' }); probeQueue.push({ kind: 'err', message: 'i/o timeout' }); const { w } = loadWatcher(); const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] }; w.healthChecker = fakeHealthChecker; await w.scanSites(); const u = w.upstreams.values().next().value; u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); await w._probeOne(u); await w._probeOne(u); await w._probeOne(u); expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1); }); test('recovery resolves the open incident after RESOLVED_AFTER_MS', async () => { seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' }); probeQueue.push({ kind: 'err', message: 'i/o timeout' }); // trip dead probeQueue.push({ kind: 'ok', statusCode: 200 }); // recovery const { w } = loadWatcher(); const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] }; w.healthChecker = fakeHealthChecker; await w.scanSites(); const u = w.upstreams.values().next().value; // Trip the dead state u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); await w._probeOne(u); expect(w.openIncidents.has('1.1.1.1:80')).toBe(true); // Simulate a recovery — lastFailureAt is 2 min ago, now healthy u.lastFailureAt = new Date(Date.now() - 2 * 60 * 1000).toISOString(); await w._probeOne(u); expect(fakeHealthChecker.resolveIncident).toHaveBeenCalledTimes(1); expect(w.openIncidents.has('1.1.1.1:80')).toBe(false); }); test('mute suppresses probing and hides upstream in snapshot status', async () => { seedSites({ 'noisy.sami': 'noisy.sami { reverse_proxy 1.1.1.1:80 }\n' }); const { w } = loadWatcher(); await w.scanSites(); w.setMuted('1.1.1.1:80', true); expect(w.isMuted('1.1.1.1:80')).toBe(true); const snap = w.snapshot(); expect(snap.upstreams[0].status).toBe('muted'); expect(snap.upstreams[0].muted).toBe(true); // probe tick should skip muted await w._tick(); // lastCheckedAt should NOT have advanced because no probe was issued expect(snap.upstreams[0].lastCheckedAt).toBeNull(); }); test('unmute resets failure counters so a recently-recovered upstream is not immediately re-incidented', async () => { seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' }); const { w } = loadWatcher(); await w.scanSites(); const u = w.upstreams.values().next().value; u.consecutiveFailures = 42; u.lastError = 'old failure'; u.lastFailureAt = new Date().toISOString(); u.status = 'down'; w.setMuted('1.1.1.1:80', true); w.setMuted('1.1.1.1:80', false); expect(u.consecutiveFailures).toBe(0); expect(u.status).toBe('unknown'); expect(u.lastError).toBeNull(); }); test('snapshot sorts dead > down > muted > up > unknown', async () => { seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n', 'b.sami': 'b.sami { reverse_proxy 2.2.2.2:80 }\n', 'c.sami': 'c.sami { reverse_proxy 3.3.3.3:80 }\n', 'd.sami': 'd.sami { reverse_proxy 4.4.4.4:80 }\n', 'e.sami': 'e.sami { reverse_proxy 5.5.5.5:80 }\n' }); const { w } = loadWatcher(); await w.scanSites(); const all = Array.from(w.upstreams.values()); // 1.1.1.1:80 -> up (just succeeded) all.find(u => u.host === '1.1.1.1:80').status = 'up'; all.find(u => u.host === '1.1.1.1:80').lastSuccessAt = new Date().toISOString(); // 2.2.2.2:80 -> down (recent — last success 30s ago) all.find(u => u.host === '2.2.2.2:80').status = 'down'; all.find(u => u.host === '2.2.2.2:80').lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 3.3.3.3:80 -> muted w.muted.add('3.3.3.3:80'); // 4.4.4.4:80 -> dead (last success 7min ago, never recovered) const dead = all.find(u => u.host === '4.4.4.4:80'); dead.status = 'down'; dead.lastSuccessAt = new Date(Date.now() - 7 * 60 * 1000).toISOString(); // 5.5.5.5:80 -> unknown (no probes yet) const snap = w.snapshot(); const order = snap.upstreams.map(u => u.host); // Expected: dead first, then down, then muted, then up, then unknown expect(order).toEqual(['4.4.4.4:80', '2.2.2.2:80', '3.3.3.3:80', '1.1.1.1:80', '5.5.5.5:80']); }); test('persists muted list to state file', async () => { seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' }); const { w } = loadWatcher(); await w.scanSites(); w.setMuted('1.1.1.1:80', true); // _saveState writes to STATE + '.tmp' then renames. Look for the .tmp // write since that's the actual writeFileSync call (rename is silent). const writes = fsState.writeLog.filter(w => w.p === STATE + '.tmp' || w.p === STATE); expect(writes.length).toBeGreaterThan(0); const last = writes[writes.length - 1]; const data = JSON.parse(last.content); expect(data.muted).toContain('1.1.1.1:80'); }); test('reload from state file restores muted list', async () => { // Pre-seed a state file with a muted host fsState.files[STATE] = JSON.stringify({ muted: ['99.99.99.99:80'], upstreams: { '99.99.99.99:80': { ip: '99.99.99.99', port: '80', site: 'old.sami', siteFile: 'old.sami' } } }); fsState.exists[STATE] = true; // And the matching site file seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' }); process.env.CADDY_UPSTREAMS_STATE_FILE = STATE; process.env.CADDY_SITES_DIR = SITES; const mod = require('../src/monitoring/caddy-upstream-watcher'); const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod; expect(w.isMuted('99.99.99.99:80')).toBe(true); }); });