feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]
Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:
- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
reverse_proxy directive, probes each upstream every 60s independent of
Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
failures via the existing healthChecker. Mute list persisted to
data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.
GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:
1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
silently skipped real prod filenames like zap.sami-ahmed.net,
samitest.space, blocks.cryptographic-triangles.org where the file
extension is .net/.space/.org. Replaced with positive filter that
excludes README/.bak/.swp/Caddyfile + content pre-check
(must contain 'reverse_proxy'). Added test covering the prod filenames.
2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
MUTED the host because the bare route used `muted !== false` which is
true for the string 'false'. Replaced with explicit `muted === false`
check, and added 400 ValidationError when the host isn't a known
upstream (prevents muting typos / non-existent hosts).
Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Smoke tests for the caddy-upstreams router.
|
||||
*
|
||||
* No jest.mock('fs') here — the route module needs a real express
|
||||
* context to load, and the watcher logic is tested separately in
|
||||
* caddy-upstream-watcher.test.js.
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
|
||||
describe('routes/caddy-upstreams', () => {
|
||||
test('router builds with all expected paths and handlers', () => {
|
||||
const mod = require('../../routes/caddy-upstreams');
|
||||
const fakeWatcher = {
|
||||
snapshot: jest.fn(() => ({ upstreams: [], config: {} }))
|
||||
};
|
||||
const fakeHealthChecker = { incidents: [] };
|
||||
|
||||
const router = mod({
|
||||
asyncHandler: (fn) => fn,
|
||||
caddyUpstreamWatcher: fakeWatcher,
|
||||
healthChecker: fakeHealthChecker
|
||||
});
|
||||
|
||||
expect(router).toBeDefined();
|
||||
expect(typeof router.use).toBe('function');
|
||||
|
||||
const paths = router.stack
|
||||
.filter((l) => l.route)
|
||||
.map((l) => Object.keys(l.route.methods).map((m) => `${m.toUpperCase()} ${l.route.path}`))
|
||||
.flat();
|
||||
|
||||
expect(paths).toEqual(expect.arrayContaining([
|
||||
'GET /caddy/upstreams',
|
||||
'GET /caddy/upstreams/incidents',
|
||||
'POST /caddy/upstreams/mute',
|
||||
'POST /caddy/upstreams/:host/mute',
|
||||
'POST /caddy/upstreams/:host/unmute'
|
||||
]));
|
||||
});
|
||||
|
||||
test('GET /caddy/upstreams responds with watcher snapshot', async () => {
|
||||
const mod = require('../../routes/caddy-upstreams');
|
||||
const fakeSnapshot = { upstreams: [{ host: '1.1.1.1:80', status: 'up', muted: false }], config: {} };
|
||||
const fakeWatcher = { snapshot: jest.fn(() => fakeSnapshot) };
|
||||
const fakeHealthChecker = { incidents: [] };
|
||||
|
||||
// Build a tiny express app with the route + a shim success/error responder.
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, res, next) => {
|
||||
res.success = (data) => res.json({ success: true, ...data });
|
||||
res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg });
|
||||
next();
|
||||
});
|
||||
app.use(mod({
|
||||
asyncHandler: (fn, _ctx) => async (req, res, next) => {
|
||||
try { await fn(req, res, next); } catch (e) { next(e); }
|
||||
},
|
||||
caddyUpstreamWatcher: fakeWatcher,
|
||||
healthChecker: fakeHealthChecker
|
||||
}));
|
||||
|
||||
const server = app.listen(0);
|
||||
const { port } = server.address();
|
||||
const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams`);
|
||||
const body = await res.json();
|
||||
server.close();
|
||||
expect(body.success).toBe(true);
|
||||
expect(body.upstreams).toEqual(fakeSnapshot.upstreams);
|
||||
});
|
||||
|
||||
test('POST /caddy/upstreams/mute with body {host, muted:"false"} does NOT mute (string coercion)', async () => {
|
||||
// Regression: bare route previously used `muted !== false` which muted
|
||||
// when muted was a string 'false' (because 'false' !== false). Fix
|
||||
// requires explicit `muted === false` to unmute.
|
||||
const mod = require('../../routes/caddy-upstreams');
|
||||
const fakeSnapshot = { upstreams: [], config: {} };
|
||||
const fakeWatcher = {
|
||||
snapshot: jest.fn(() => fakeSnapshot),
|
||||
upstreams: new Map([['known:80', { host: 'known:80' }]]),
|
||||
setMuted: jest.fn(() => ({ host: 'known:80', muted: false }))
|
||||
};
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, res, next) => {
|
||||
res.success = (data) => res.json({ success: true, ...data });
|
||||
res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg });
|
||||
next();
|
||||
});
|
||||
app.use(mod({
|
||||
asyncHandler: (fn, _ctx) => async (req, res, next) => {
|
||||
try { await fn(req, res, next); } catch (e) { next(e); }
|
||||
},
|
||||
caddyUpstreamWatcher: fakeWatcher,
|
||||
healthChecker: { incidents: [] }
|
||||
}));
|
||||
// Error middleware MUST be registered AFTER routes so it actually catches.
|
||||
app.use((err, req, res, next) => {
|
||||
if (err && err.statusCode === 400) {
|
||||
return res.status(400).json({ success: false, error: err.message });
|
||||
}
|
||||
return res.status(err?.statusCode || 500).json({ success: false, error: err?.message || 'unknown' });
|
||||
});
|
||||
|
||||
const server = app.listen(0);
|
||||
const { port } = server.address();
|
||||
|
||||
// String 'false' should NOT mute (should unmute or pass through)
|
||||
const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ host: 'known:80', muted: 'false' })
|
||||
});
|
||||
const body = await res.json();
|
||||
expect(fakeWatcher.setMuted).toHaveBeenCalledWith('known:80', false);
|
||||
|
||||
// Unknown host should 400
|
||||
fakeWatcher.setMuted.mockClear();
|
||||
const res2 = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ host: 'not-a-real-host:80' })
|
||||
});
|
||||
const body2 = await res2.json();
|
||||
server.close();
|
||||
expect(res2.status).toBe(400);
|
||||
expect(body2.error).toMatch(/not a known upstream/);
|
||||
expect(fakeWatcher.setMuted).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user