sami7777/dashcaddy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]
CI / Test & Lint (push) Canceled after 0s
Details
CI / Security audit (push) Canceled after 0s
Details

Browse Source 
Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:

- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
  reverse_proxy directive, probes each upstream every 60s independent of
  Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
  failures via the existing healthChecker. Mute list persisted to
  data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
  forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
  GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
  POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
  POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
  All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.

GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:

1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
   silently skipped real prod filenames like zap.sami-ahmed.net,
   samitest.space, blocks.cryptographic-triangles.org where the file
   extension is .net/.space/.org. Replaced with positive filter that
   excludes README/.bak/.swp/Caddyfile + content pre-check
   (must contain 'reverse_proxy'). Added test covering the prod filenames.

2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
   MUTED the host because the bare route used `muted !== false` which is
   true for the string 'false'. Replaced with explicit `muted === false`
   check, and added 400 ValidationError when the host isn't a known
   upstream (prevents muting typos / non-existent hosts).

Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.
This commit is contained in:
Sami Ahmed
2026-08-17 17:11:27 -07:00
parent 6d875e4631
commit 45cfa83bad
5 changed files with 1090 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashcaddy-api/__tests__/caddy-upstream-watcher.test.js
+389
View File
@@ -0,0 +1,389 @@
/**
* Tests for caddy-upstream-watcher.
*
* Mock-driven: we stub fs (for /etc/caddy/sites scan + state file) and http/https
* (for the probe). Tests cover site parsing, probe happy/sad path, the 5-minute
* "dead" threshold, mute toggle, and incident integration with healthChecker.
*/
const path = require('path');
const Module = require('module');
// Mock fs with controllable behavior.
const fsState = {
files: {}, // path -> string content
exists: {}, // path -> bool
writeLog: [], // writes
};
jest.mock('fs', () => {
const real = jest.requireActual('fs');
return {
...real,
existsSync: jest.fn((p) => fsState.exists[p] !== undefined ? fsState.exists[p] : (fsState.files[p] !== undefined)),
readFileSync: jest.fn((p) => {
if (fsState.files[p] === undefined) {
const e = new Error(`ENOENT: ${p}`);
e.code = 'ENOENT';
throw e;
}
return fsState.files[p];
}),
readdirSync: jest.fn((p) => Object.keys(fsState.files).filter(f => f.startsWith(p + '/')).map(f => f.substring(p.length + 1))),
writeFileSync: jest.fn((p, content) => {
fsState.writeLog.push({ p, content });
fsState.files[p] = content;
fsState.exists[p] = true;
}),
mkdirSync: jest.fn(),
renameSync: jest.fn((src, dst) => {
fsState.files[dst] = fsState.files[src];
fsState.exists[dst] = true;
delete fsState.files[src];
delete fsState.exists[src];
})
};
});
// Mock http/https request to control probe responses.
const probeQueue = []; // each entry: { kind: 'ok'|'err'|'timeout'|'code', statusCode? }
jest.mock('http', () => ({
request: jest.fn((opts, cb) => {
const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
const handlers = {};
const res = {
statusCode: entry.statusCode || 200,
headers: { server: 'mock' },
resume: () => {},
on: (e, fn) => { handlers[e] = fn; }
};
const req = {
on: jest.fn((e, fn) => { handlers[e] = fn; }),
end: jest.fn(() => {
if (entry.kind === 'err') {
handlers.error && handlers.error(new Error(entry.message || 'connect ECONNREFUSED'));
return;
}
if (entry.kind === 'timeout') {
handlers.timeout && handlers.timeout();
return;
}
cb(res);
if (handlers.end) handlers.end();
}),
destroy: jest.fn()
};
return req;
})
}));
jest.mock('https', () => ({
request: jest.fn((opts, cb) => {
const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
const handlers = {};
const res = {
statusCode: entry.statusCode || 200,
headers: { server: 'mock-https' },
resume: () => {},
on: (e, fn) => { handlers[e] = fn; }
};
const req = {
on: jest.fn((e, fn) => { handlers[e] = fn; }),
end: jest.fn(() => {
if (entry.kind === 'err') {
handlers.error && handlers.error(new Error(entry.message || 'TLS error'));
return;
}
cb(res);
if (handlers.end) handlers.end();
}),
destroy: jest.fn()
};
return req;
})
}));
// Reset fs mock state between tests.
beforeEach(() => {
fsState.files = {};
fsState.exists = {};
fsState.writeLog = [];
probeQueue.length = 0;
jest.clearAllMocks();
jest.resetModules();
});
describe('CaddyUpstreamWatcher', () => {
const SITES = '/etc/caddy/sites';
const STATE = '/tmp/caddy-upstreams-test.json';
function seedSites(files) {
for (const [name, content] of Object.entries(files)) {
fsState.files[SITES + '/' + name] = content;
fsState.exists[SITES + '/' + name] = true;
}
}
function loadWatcher() {
process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
process.env.CADDY_SITES_DIR = SITES;
// Disable the singleton's auto-write so we can call _saveState manually.
const mod = require('../src/monitoring/caddy-upstream-watcher');
return { mod, w: mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod };
}
test('parses reverse_proxy directives from /etc/caddy/sites/*', async () => {
seedSites({
'arch.sami': `arch.sami {\n reverse_proxy 100.120.159.34:5000 { health_uri /api/stats }\n}`,
'appt.sami': `appt.sami {\n reverse_proxy http://100.81.59.99:5232\n}`,
'zap.sami': `zap.sami {\n reverse_proxy 10.0.0.5:8080\n reverse_proxy 10.0.0.6:8080 # multiple upstreams in same site block\n}`
});
const { w } = loadWatcher();
await w.scanSites();
const snap = w.snapshot();
const hosts = snap.upstreams.map(u => u.host).sort();
expect(hosts).toEqual(['10.0.0.5:8080', '10.0.0.6:8080', '100.120.159.34:5000', '100.81.59.99:5232']);
expect(snap.upstreams.find(u => u.host === '100.120.159.34:5000').site).toBe('arch.sami');
expect(snap.upstreams.find(u => u.host === '100.81.59.99:5232').site).toBe('appt.sami');
});
test('ignores non-site files and unparseable entries', async () => {
seedSites({
'README.md': '# documentation\nreverse_proxy 1.2.3.4:9999\n', // not a site file
'garbage.sami': 'not a caddyfile\n', // no reverse_proxy
'good.sami': 'good.sami {\n reverse_proxy 1.2.3.4:9999\n}\n'
});
const { w } = loadWatcher();
await w.scanSites();
const hosts = w.snapshot().upstreams.map(u => u.host);
expect(hosts).toEqual(['1.2.3.4:9999']);
});
test('handles real prod-style filenames: zap.sami-ahmed.net, samitest.space, blocks.cryptographic-triangles.org', async () => {
// These are the actual file names in production /etc/caddy/sites/ —
// extension is .net / .space / .org, NOT .sami/.caddy/.conf. The old
// file-extension filter would skip them silently.
seedSites({
'zap.sami-ahmed.net': 'zap.sami-ahmed.net {\n reverse_proxy localhost:8088\n}\n',
'samitest.space': 'samitest.space {\n reverse_proxy 100.120.159.34:8080 {}\n}\n',
'blocks.cryptographic-triangles.org': 'blocks.cryptographic-triangles.org {\n\treverse_proxy localhost:3052 {}\n}\n'
});
const { w } = loadWatcher();
await w.scanSites();
const snap = w.snapshot();
const byHost = Object.fromEntries(snap.upstreams.map(u => [u.host, u.site]));
expect(byHost['localhost:8088']).toBe('zap.sami-ahmed.net');
expect(byHost['100.120.159.34:8080']).toBe('samitest.space');
expect(byHost['localhost:3052']).toBe('blocks.cryptographic-triangles.org');
});
test('drops upstreams that disappear from the sites dir', async () => {
seedSites({
'arch.sami': 'arch.sami {\n reverse_proxy 1.1.1.1:5000\n}\n'
});
const { w } = loadWatcher();
await w.scanSites();
expect(w.upstreams.size).toBe(1);
fsState.files = {}; // wipe
fsState.exists = {};
await w.scanSites();
expect(w.upstreams.size).toBe(0);
});
test('healthy probe updates state and does not open an incident', async () => {
seedSites({ 'good.sami': 'good.sami { reverse_proxy 1.1.1.1:80 }\n' });
probeQueue.push({ kind: 'ok', statusCode: 200 });
const { w } = loadWatcher();
const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
w.healthChecker = fakeHealthChecker;
await w.scanSites();
await w._probeOne(w.upstreams.values().next().value);
const snap = w.snapshot();
expect(snap.upstreams[0].status).toBe('up');
expect(snap.upstreams[0].lastSuccessAt).toBeTruthy();
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
});
test('auth-walled 4xx counts as healthy (proves the upstream answered)', async () => {
seedSites({ 'auth.sami': 'auth.sami { reverse_proxy 1.1.1.1:80 }\n' });
probeQueue.push({ kind: 'ok', statusCode: 401 });
const { w } = loadWatcher();
await w.scanSites();
await w._probeOne(w.upstreams.values().next().value);
expect(w.snapshot().upstreams[0].status).toBe('up');
});
test('first failure flips status to down but does NOT open an incident (under 5min)', async () => {
seedSites({ 'bad.sami': 'bad.sami { reverse_proxy 1.1.1.1:80 }\n' });
probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED' });
const { w } = loadWatcher();
const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
w.healthChecker = fakeHealthChecker;
await w.scanSites();
const u = w.upstreams.values().next().value;
u.lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 30s ago it was healthy
await w._probeOne(u);
const snap = w.snapshot();
expect(snap.upstreams[0].status).toBe('down');
expect(snap.upstreams[0].failingForMs).toBeLessThan(5 * 60 * 1000);
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
});
test('after 5 minutes of consecutive failures an incident is opened', async () => {
seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
const { w } = loadWatcher();
const incidents = [];
const fakeHealthChecker = {
createIncident: jest.fn((serviceId, type, message, status) => {
incidents.push({ serviceId, type, message, status });
}),
incidents: []
};
w.healthChecker = fakeHealthChecker;
await w.scanSites();
const u = w.upstreams.values().next().value;
// Simulate lastSuccessAt being 6 minutes ago so failingForMs exceeds DEAD_AFTER_MS.
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
await w._probeOne(u);
expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
expect(fakeHealthChecker.createIncident.mock.calls[0][1]).toBe('caddy-upstream-dead');
expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
});
test('does not duplicate incidents for the same upstream', async () => {
seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
// Queue up 3 errors so each probe fails.
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
probeQueue.push({ kind: 'err', message: 'i/o timeout' });
const { w } = loadWatcher();
const fakeHealthChecker = {
createIncident: jest.fn(),
incidents: []
};
w.healthChecker = fakeHealthChecker;
await w.scanSites();
const u = w.upstreams.values().next().value;
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
await w._probeOne(u);
await w._probeOne(u);
await w._probeOne(u);
expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
});
test('recovery resolves the open incident after RESOLVED_AFTER_MS', async () => {
seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
probeQueue.push({ kind: 'err', message: 'i/o timeout' }); // trip dead
probeQueue.push({ kind: 'ok', statusCode: 200 }); // recovery
const { w } = loadWatcher();
const fakeHealthChecker = {
createIncident: jest.fn(),
resolveIncident: jest.fn(),
incidents: []
};
w.healthChecker = fakeHealthChecker;
await w.scanSites();
const u = w.upstreams.values().next().value;
// Trip the dead state
u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
await w._probeOne(u);
expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
// Simulate a recovery — lastFailureAt is 2 min ago, now healthy
u.lastFailureAt = new Date(Date.now() - 2 * 60 * 1000).toISOString();
await w._probeOne(u);
expect(fakeHealthChecker.resolveIncident).toHaveBeenCalledTimes(1);
expect(w.openIncidents.has('1.1.1.1:80')).toBe(false);
});
test('mute suppresses probing and hides upstream in snapshot status', async () => {
seedSites({ 'noisy.sami': 'noisy.sami { reverse_proxy 1.1.1.1:80 }\n' });
const { w } = loadWatcher();
await w.scanSites();
w.setMuted('1.1.1.1:80', true);
expect(w.isMuted('1.1.1.1:80')).toBe(true);
const snap = w.snapshot();
expect(snap.upstreams[0].status).toBe('muted');
expect(snap.upstreams[0].muted).toBe(true);
// probe tick should skip muted
await w._tick();
// lastCheckedAt should NOT have advanced because no probe was issued
expect(snap.upstreams[0].lastCheckedAt).toBeNull();
});
test('unmute resets failure counters so a recently-recovered upstream is not immediately re-incidented', async () => {
seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
const { w } = loadWatcher();
await w.scanSites();
const u = w.upstreams.values().next().value;
u.consecutiveFailures = 42;
u.lastError = 'old failure';
u.lastFailureAt = new Date().toISOString();
u.status = 'down';
w.setMuted('1.1.1.1:80', true);
w.setMuted('1.1.1.1:80', false);
expect(u.consecutiveFailures).toBe(0);
expect(u.status).toBe('unknown');
expect(u.lastError).toBeNull();
});
test('snapshot sorts dead > down > muted > up > unknown', async () => {
seedSites({
'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
'b.sami': 'b.sami { reverse_proxy 2.2.2.2:80 }\n',
'c.sami': 'c.sami { reverse_proxy 3.3.3.3:80 }\n',
'd.sami': 'd.sami { reverse_proxy 4.4.4.4:80 }\n',
'e.sami': 'e.sami { reverse_proxy 5.5.5.5:80 }\n'
});
const { w } = loadWatcher();
await w.scanSites();
const all = Array.from(w.upstreams.values());
// 1.1.1.1:80 -> up (just succeeded)
all.find(u => u.host === '1.1.1.1:80').status = 'up';
all.find(u => u.host === '1.1.1.1:80').lastSuccessAt = new Date().toISOString();
// 2.2.2.2:80 -> down (recent — last success 30s ago)
all.find(u => u.host === '2.2.2.2:80').status = 'down';
all.find(u => u.host === '2.2.2.2:80').lastSuccessAt = new Date(Date.now() - 30000).toISOString();
// 3.3.3.3:80 -> muted
w.muted.add('3.3.3.3:80');
// 4.4.4.4:80 -> dead (last success 7min ago, never recovered)
const dead = all.find(u => u.host === '4.4.4.4:80');
dead.status = 'down';
dead.lastSuccessAt = new Date(Date.now() - 7 * 60 * 1000).toISOString();
// 5.5.5.5:80 -> unknown (no probes yet)
const snap = w.snapshot();
const order = snap.upstreams.map(u => u.host);
// Expected: dead first, then down, then muted, then up, then unknown
expect(order).toEqual(['4.4.4.4:80', '2.2.2.2:80', '3.3.3.3:80', '1.1.1.1:80', '5.5.5.5:80']);
});
test('persists muted list to state file', async () => {
seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' });
const { w } = loadWatcher();
await w.scanSites();
w.setMuted('1.1.1.1:80', true);
// _saveState writes to STATE + '.tmp' then renames. Look for the .tmp
// write since that's the actual writeFileSync call (rename is silent).
const writes = fsState.writeLog.filter(w => w.p === STATE + '.tmp' || w.p === STATE);
expect(writes.length).toBeGreaterThan(0);
const last = writes[writes.length - 1];
const data = JSON.parse(last.content);
expect(data.muted).toContain('1.1.1.1:80');
});
test('reload from state file restores muted list', async () => {
// Pre-seed a state file with a muted host
fsState.files[STATE] = JSON.stringify({
muted: ['99.99.99.99:80'],
upstreams: { '99.99.99.99:80': { ip: '99.99.99.99', port: '80', site: 'old.sami', siteFile: 'old.sami' } }
});
fsState.exists[STATE] = true;
// And the matching site file
seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' });
process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
process.env.CADDY_SITES_DIR = SITES;
const mod = require('../src/monitoring/caddy-upstream-watcher');
const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod;
expect(w.isMuted('99.99.99.99:80')).toBe(true);
});
});