diff --git a/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js b/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js new file mode 100644 index 0000000..04228ef --- /dev/null +++ b/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js @@ -0,0 +1,389 @@ +/** + * Tests for caddy-upstream-watcher. + * + * Mock-driven: we stub fs (for /etc/caddy/sites scan + state file) and http/https + * (for the probe). Tests cover site parsing, probe happy/sad path, the 5-minute + * "dead" threshold, mute toggle, and incident integration with healthChecker. + */ + +const path = require('path'); +const Module = require('module'); + +// Mock fs with controllable behavior. +const fsState = { + files: {}, // path -> string content + exists: {}, // path -> bool + writeLog: [], // writes +}; + +jest.mock('fs', () => { + const real = jest.requireActual('fs'); + return { + ...real, + existsSync: jest.fn((p) => fsState.exists[p] !== undefined ? fsState.exists[p] : (fsState.files[p] !== undefined)), + readFileSync: jest.fn((p) => { + if (fsState.files[p] === undefined) { + const e = new Error(`ENOENT: ${p}`); + e.code = 'ENOENT'; + throw e; + } + return fsState.files[p]; + }), + readdirSync: jest.fn((p) => Object.keys(fsState.files).filter(f => f.startsWith(p + '/')).map(f => f.substring(p.length + 1))), + writeFileSync: jest.fn((p, content) => { + fsState.writeLog.push({ p, content }); + fsState.files[p] = content; + fsState.exists[p] = true; + }), + mkdirSync: jest.fn(), + renameSync: jest.fn((src, dst) => { + fsState.files[dst] = fsState.files[src]; + fsState.exists[dst] = true; + delete fsState.files[src]; + delete fsState.exists[src]; + }) + }; +}); + +// Mock http/https request to control probe responses. +const probeQueue = []; // each entry: { kind: 'ok'|'err'|'timeout'|'code', statusCode? } +jest.mock('http', () => ({ + request: jest.fn((opts, cb) => { + const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 }; + const handlers = {}; + const res = { + statusCode: entry.statusCode || 200, + headers: { server: 'mock' }, + resume: () => {}, + on: (e, fn) => { handlers[e] = fn; } + }; + const req = { + on: jest.fn((e, fn) => { handlers[e] = fn; }), + end: jest.fn(() => { + if (entry.kind === 'err') { + handlers.error && handlers.error(new Error(entry.message || 'connect ECONNREFUSED')); + return; + } + if (entry.kind === 'timeout') { + handlers.timeout && handlers.timeout(); + return; + } + cb(res); + if (handlers.end) handlers.end(); + }), + destroy: jest.fn() + }; + return req; + }) +})); +jest.mock('https', () => ({ + request: jest.fn((opts, cb) => { + const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 }; + const handlers = {}; + const res = { + statusCode: entry.statusCode || 200, + headers: { server: 'mock-https' }, + resume: () => {}, + on: (e, fn) => { handlers[e] = fn; } + }; + const req = { + on: jest.fn((e, fn) => { handlers[e] = fn; }), + end: jest.fn(() => { + if (entry.kind === 'err') { + handlers.error && handlers.error(new Error(entry.message || 'TLS error')); + return; + } + cb(res); + if (handlers.end) handlers.end(); + }), + destroy: jest.fn() + }; + return req; + }) +})); + +// Reset fs mock state between tests. +beforeEach(() => { + fsState.files = {}; + fsState.exists = {}; + fsState.writeLog = []; + probeQueue.length = 0; + jest.clearAllMocks(); + jest.resetModules(); +}); + +describe('CaddyUpstreamWatcher', () => { + const SITES = '/etc/caddy/sites'; + const STATE = '/tmp/caddy-upstreams-test.json'; + + function seedSites(files) { + for (const [name, content] of Object.entries(files)) { + fsState.files[SITES + '/' + name] = content; + fsState.exists[SITES + '/' + name] = true; + } + } + + function loadWatcher() { + process.env.CADDY_UPSTREAMS_STATE_FILE = STATE; + process.env.CADDY_SITES_DIR = SITES; + // Disable the singleton's auto-write so we can call _saveState manually. + const mod = require('../src/monitoring/caddy-upstream-watcher'); + return { mod, w: mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod }; + } + + test('parses reverse_proxy directives from /etc/caddy/sites/*', async () => { + seedSites({ + 'arch.sami': `arch.sami {\n reverse_proxy 100.120.159.34:5000 { health_uri /api/stats }\n}`, + 'appt.sami': `appt.sami {\n reverse_proxy http://100.81.59.99:5232\n}`, + 'zap.sami': `zap.sami {\n reverse_proxy 10.0.0.5:8080\n reverse_proxy 10.0.0.6:8080 # multiple upstreams in same site block\n}` + }); + const { w } = loadWatcher(); + await w.scanSites(); + const snap = w.snapshot(); + const hosts = snap.upstreams.map(u => u.host).sort(); + expect(hosts).toEqual(['10.0.0.5:8080', '10.0.0.6:8080', '100.120.159.34:5000', '100.81.59.99:5232']); + expect(snap.upstreams.find(u => u.host === '100.120.159.34:5000').site).toBe('arch.sami'); + expect(snap.upstreams.find(u => u.host === '100.81.59.99:5232').site).toBe('appt.sami'); + }); + + test('ignores non-site files and unparseable entries', async () => { + seedSites({ + 'README.md': '# documentation\nreverse_proxy 1.2.3.4:9999\n', // not a site file + 'garbage.sami': 'not a caddyfile\n', // no reverse_proxy + 'good.sami': 'good.sami {\n reverse_proxy 1.2.3.4:9999\n}\n' + }); + const { w } = loadWatcher(); + await w.scanSites(); + const hosts = w.snapshot().upstreams.map(u => u.host); + expect(hosts).toEqual(['1.2.3.4:9999']); + }); + + test('handles real prod-style filenames: zap.sami-ahmed.net, samitest.space, blocks.cryptographic-triangles.org', async () => { + // These are the actual file names in production /etc/caddy/sites/ — + // extension is .net / .space / .org, NOT .sami/.caddy/.conf. The old + // file-extension filter would skip them silently. + seedSites({ + 'zap.sami-ahmed.net': 'zap.sami-ahmed.net {\n reverse_proxy localhost:8088\n}\n', + 'samitest.space': 'samitest.space {\n reverse_proxy 100.120.159.34:8080 {}\n}\n', + 'blocks.cryptographic-triangles.org': 'blocks.cryptographic-triangles.org {\n\treverse_proxy localhost:3052 {}\n}\n' + }); + const { w } = loadWatcher(); + await w.scanSites(); + const snap = w.snapshot(); + const byHost = Object.fromEntries(snap.upstreams.map(u => [u.host, u.site])); + expect(byHost['localhost:8088']).toBe('zap.sami-ahmed.net'); + expect(byHost['100.120.159.34:8080']).toBe('samitest.space'); + expect(byHost['localhost:3052']).toBe('blocks.cryptographic-triangles.org'); + }); + + test('drops upstreams that disappear from the sites dir', async () => { + seedSites({ + 'arch.sami': 'arch.sami {\n reverse_proxy 1.1.1.1:5000\n}\n' + }); + const { w } = loadWatcher(); + await w.scanSites(); + expect(w.upstreams.size).toBe(1); + fsState.files = {}; // wipe + fsState.exists = {}; + await w.scanSites(); + expect(w.upstreams.size).toBe(0); + }); + + test('healthy probe updates state and does not open an incident', async () => { + seedSites({ 'good.sami': 'good.sami { reverse_proxy 1.1.1.1:80 }\n' }); + probeQueue.push({ kind: 'ok', statusCode: 200 }); + const { w } = loadWatcher(); + const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + await w._probeOne(w.upstreams.values().next().value); + const snap = w.snapshot(); + expect(snap.upstreams[0].status).toBe('up'); + expect(snap.upstreams[0].lastSuccessAt).toBeTruthy(); + expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled(); + }); + + test('auth-walled 4xx counts as healthy (proves the upstream answered)', async () => { + seedSites({ 'auth.sami': 'auth.sami { reverse_proxy 1.1.1.1:80 }\n' }); + probeQueue.push({ kind: 'ok', statusCode: 401 }); + const { w } = loadWatcher(); + await w.scanSites(); + await w._probeOne(w.upstreams.values().next().value); + expect(w.snapshot().upstreams[0].status).toBe('up'); + }); + + test('first failure flips status to down but does NOT open an incident (under 5min)', async () => { + seedSites({ 'bad.sami': 'bad.sami { reverse_proxy 1.1.1.1:80 }\n' }); + probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED' }); + const { w } = loadWatcher(); + const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + const u = w.upstreams.values().next().value; + u.lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 30s ago it was healthy + await w._probeOne(u); + const snap = w.snapshot(); + expect(snap.upstreams[0].status).toBe('down'); + expect(snap.upstreams[0].failingForMs).toBeLessThan(5 * 60 * 1000); + expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled(); + }); + + test('after 5 minutes of consecutive failures an incident is opened', async () => { + seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' }); + probeQueue.push({ kind: 'err', message: 'i/o timeout' }); + const { w } = loadWatcher(); + const incidents = []; + const fakeHealthChecker = { + createIncident: jest.fn((serviceId, type, message, status) => { + incidents.push({ serviceId, type, message, status }); + }), + incidents: [] + }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + const u = w.upstreams.values().next().value; + // Simulate lastSuccessAt being 6 minutes ago so failingForMs exceeds DEAD_AFTER_MS. + u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); + await w._probeOne(u); + expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1); + expect(fakeHealthChecker.createIncident.mock.calls[0][1]).toBe('caddy-upstream-dead'); + expect(w.openIncidents.has('1.1.1.1:80')).toBe(true); + }); + + test('does not duplicate incidents for the same upstream', async () => { + seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' }); + // Queue up 3 errors so each probe fails. + probeQueue.push({ kind: 'err', message: 'i/o timeout' }); + probeQueue.push({ kind: 'err', message: 'i/o timeout' }); + probeQueue.push({ kind: 'err', message: 'i/o timeout' }); + const { w } = loadWatcher(); + const fakeHealthChecker = { + createIncident: jest.fn(), + incidents: [] + }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + const u = w.upstreams.values().next().value; + u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); + await w._probeOne(u); + await w._probeOne(u); + await w._probeOne(u); + expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1); + }); + + test('recovery resolves the open incident after RESOLVED_AFTER_MS', async () => { + seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' }); + probeQueue.push({ kind: 'err', message: 'i/o timeout' }); // trip dead + probeQueue.push({ kind: 'ok', statusCode: 200 }); // recovery + const { w } = loadWatcher(); + const fakeHealthChecker = { + createIncident: jest.fn(), + resolveIncident: jest.fn(), + incidents: [] + }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + const u = w.upstreams.values().next().value; + // Trip the dead state + u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString(); + await w._probeOne(u); + expect(w.openIncidents.has('1.1.1.1:80')).toBe(true); + // Simulate a recovery — lastFailureAt is 2 min ago, now healthy + u.lastFailureAt = new Date(Date.now() - 2 * 60 * 1000).toISOString(); + await w._probeOne(u); + expect(fakeHealthChecker.resolveIncident).toHaveBeenCalledTimes(1); + expect(w.openIncidents.has('1.1.1.1:80')).toBe(false); + }); + + test('mute suppresses probing and hides upstream in snapshot status', async () => { + seedSites({ 'noisy.sami': 'noisy.sami { reverse_proxy 1.1.1.1:80 }\n' }); + const { w } = loadWatcher(); + await w.scanSites(); + w.setMuted('1.1.1.1:80', true); + expect(w.isMuted('1.1.1.1:80')).toBe(true); + const snap = w.snapshot(); + expect(snap.upstreams[0].status).toBe('muted'); + expect(snap.upstreams[0].muted).toBe(true); + // probe tick should skip muted + await w._tick(); + // lastCheckedAt should NOT have advanced because no probe was issued + expect(snap.upstreams[0].lastCheckedAt).toBeNull(); + }); + + test('unmute resets failure counters so a recently-recovered upstream is not immediately re-incidented', async () => { + seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' }); + const { w } = loadWatcher(); + await w.scanSites(); + const u = w.upstreams.values().next().value; + u.consecutiveFailures = 42; + u.lastError = 'old failure'; + u.lastFailureAt = new Date().toISOString(); + u.status = 'down'; + w.setMuted('1.1.1.1:80', true); + w.setMuted('1.1.1.1:80', false); + expect(u.consecutiveFailures).toBe(0); + expect(u.status).toBe('unknown'); + expect(u.lastError).toBeNull(); + }); + + test('snapshot sorts dead > down > muted > up > unknown', async () => { + seedSites({ + 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n', + 'b.sami': 'b.sami { reverse_proxy 2.2.2.2:80 }\n', + 'c.sami': 'c.sami { reverse_proxy 3.3.3.3:80 }\n', + 'd.sami': 'd.sami { reverse_proxy 4.4.4.4:80 }\n', + 'e.sami': 'e.sami { reverse_proxy 5.5.5.5:80 }\n' + }); + const { w } = loadWatcher(); + await w.scanSites(); + const all = Array.from(w.upstreams.values()); + // 1.1.1.1:80 -> up (just succeeded) + all.find(u => u.host === '1.1.1.1:80').status = 'up'; + all.find(u => u.host === '1.1.1.1:80').lastSuccessAt = new Date().toISOString(); + // 2.2.2.2:80 -> down (recent — last success 30s ago) + all.find(u => u.host === '2.2.2.2:80').status = 'down'; + all.find(u => u.host === '2.2.2.2:80').lastSuccessAt = new Date(Date.now() - 30000).toISOString(); + // 3.3.3.3:80 -> muted + w.muted.add('3.3.3.3:80'); + // 4.4.4.4:80 -> dead (last success 7min ago, never recovered) + const dead = all.find(u => u.host === '4.4.4.4:80'); + dead.status = 'down'; + dead.lastSuccessAt = new Date(Date.now() - 7 * 60 * 1000).toISOString(); + // 5.5.5.5:80 -> unknown (no probes yet) + const snap = w.snapshot(); + const order = snap.upstreams.map(u => u.host); + // Expected: dead first, then down, then muted, then up, then unknown + expect(order).toEqual(['4.4.4.4:80', '2.2.2.2:80', '3.3.3.3:80', '1.1.1.1:80', '5.5.5.5:80']); + }); + + test('persists muted list to state file', async () => { + seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' }); + const { w } = loadWatcher(); + await w.scanSites(); + w.setMuted('1.1.1.1:80', true); + // _saveState writes to STATE + '.tmp' then renames. Look for the .tmp + // write since that's the actual writeFileSync call (rename is silent). + const writes = fsState.writeLog.filter(w => w.p === STATE + '.tmp' || w.p === STATE); + expect(writes.length).toBeGreaterThan(0); + const last = writes[writes.length - 1]; + const data = JSON.parse(last.content); + expect(data.muted).toContain('1.1.1.1:80'); + }); + + test('reload from state file restores muted list', async () => { + // Pre-seed a state file with a muted host + fsState.files[STATE] = JSON.stringify({ + muted: ['99.99.99.99:80'], + upstreams: { '99.99.99.99:80': { ip: '99.99.99.99', port: '80', site: 'old.sami', siteFile: 'old.sami' } } + }); + fsState.exists[STATE] = true; + // And the matching site file + seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' }); + + process.env.CADDY_UPSTREAMS_STATE_FILE = STATE; + process.env.CADDY_SITES_DIR = SITES; + const mod = require('../src/monitoring/caddy-upstream-watcher'); + const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod; + expect(w.isMuted('99.99.99.99:80')).toBe(true); + }); +}); \ No newline at end of file diff --git a/dashcaddy-api/__tests__/routes/caddy-upstreams.routes.test.js b/dashcaddy-api/__tests__/routes/caddy-upstreams.routes.test.js new file mode 100644 index 0000000..fc24477 --- /dev/null +++ b/dashcaddy-api/__tests__/routes/caddy-upstreams.routes.test.js @@ -0,0 +1,132 @@ +/** + * Smoke tests for the caddy-upstreams router. + * + * No jest.mock('fs') here — the route module needs a real express + * context to load, and the watcher logic is tested separately in + * caddy-upstream-watcher.test.js. + */ + +const express = require('express'); + +describe('routes/caddy-upstreams', () => { + test('router builds with all expected paths and handlers', () => { + const mod = require('../../routes/caddy-upstreams'); + const fakeWatcher = { + snapshot: jest.fn(() => ({ upstreams: [], config: {} })) + }; + const fakeHealthChecker = { incidents: [] }; + + const router = mod({ + asyncHandler: (fn) => fn, + caddyUpstreamWatcher: fakeWatcher, + healthChecker: fakeHealthChecker + }); + + expect(router).toBeDefined(); + expect(typeof router.use).toBe('function'); + + const paths = router.stack + .filter((l) => l.route) + .map((l) => Object.keys(l.route.methods).map((m) => `${m.toUpperCase()} ${l.route.path}`)) + .flat(); + + expect(paths).toEqual(expect.arrayContaining([ + 'GET /caddy/upstreams', + 'GET /caddy/upstreams/incidents', + 'POST /caddy/upstreams/mute', + 'POST /caddy/upstreams/:host/mute', + 'POST /caddy/upstreams/:host/unmute' + ])); + }); + + test('GET /caddy/upstreams responds with watcher snapshot', async () => { + const mod = require('../../routes/caddy-upstreams'); + const fakeSnapshot = { upstreams: [{ host: '1.1.1.1:80', status: 'up', muted: false }], config: {} }; + const fakeWatcher = { snapshot: jest.fn(() => fakeSnapshot) }; + const fakeHealthChecker = { incidents: [] }; + + // Build a tiny express app with the route + a shim success/error responder. + const app = express(); + app.use(express.json()); + app.use((req, res, next) => { + res.success = (data) => res.json({ success: true, ...data }); + res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg }); + next(); + }); + app.use(mod({ + asyncHandler: (fn, _ctx) => async (req, res, next) => { + try { await fn(req, res, next); } catch (e) { next(e); } + }, + caddyUpstreamWatcher: fakeWatcher, + healthChecker: fakeHealthChecker + })); + + const server = app.listen(0); + const { port } = server.address(); + const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams`); + const body = await res.json(); + server.close(); + expect(body.success).toBe(true); + expect(body.upstreams).toEqual(fakeSnapshot.upstreams); + }); + + test('POST /caddy/upstreams/mute with body {host, muted:"false"} does NOT mute (string coercion)', async () => { + // Regression: bare route previously used `muted !== false` which muted + // when muted was a string 'false' (because 'false' !== false). Fix + // requires explicit `muted === false` to unmute. + const mod = require('../../routes/caddy-upstreams'); + const fakeSnapshot = { upstreams: [], config: {} }; + const fakeWatcher = { + snapshot: jest.fn(() => fakeSnapshot), + upstreams: new Map([['known:80', { host: 'known:80' }]]), + setMuted: jest.fn(() => ({ host: 'known:80', muted: false })) + }; + + const app = express(); + app.use(express.json()); + app.use((req, res, next) => { + res.success = (data) => res.json({ success: true, ...data }); + res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg }); + next(); + }); + app.use(mod({ + asyncHandler: (fn, _ctx) => async (req, res, next) => { + try { await fn(req, res, next); } catch (e) { next(e); } + }, + caddyUpstreamWatcher: fakeWatcher, + healthChecker: { incidents: [] } + })); + // Error middleware MUST be registered AFTER routes so it actually catches. + app.use((err, req, res, next) => { + if (err && err.statusCode === 400) { + return res.status(400).json({ success: false, error: err.message }); + } + return res.status(err?.statusCode || 500).json({ success: false, error: err?.message || 'unknown' }); + }); + + const server = app.listen(0); + const { port } = server.address(); + + // String 'false' should NOT mute (should unmute or pass through) + const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ host: 'known:80', muted: 'false' }) + }); + const body = await res.json(); + expect(fakeWatcher.setMuted).toHaveBeenCalledWith('known:80', false); + + // Unknown host should 400 + fakeWatcher.setMuted.mockClear(); + const res2 = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ host: 'not-a-real-host:80' }) + }); + const body2 = await res2.json(); + server.close(); + expect(res2.status).toBe(400); + expect(body2.error).toMatch(/not a known upstream/); + expect(fakeWatcher.setMuted).not.toHaveBeenCalled(); + }); +}); \ No newline at end of file diff --git a/dashcaddy-api/routes/caddy-upstreams.js b/dashcaddy-api/routes/caddy-upstreams.js new file mode 100644 index 0000000..851d8cc --- /dev/null +++ b/dashcaddy-api/routes/caddy-upstreams.js @@ -0,0 +1,109 @@ +/** + * Caddy upstreams routes + * + * Exposes: + * GET /api/v1/caddy/upstreams — full snapshot + * GET /api/v1/caddy/upstreams/incidents — open dead-upstream incidents (via healthChecker) + * POST /api/v1/caddy/upstreams/:host/mute — body { muted: true|false } (also via query ?muted=true) + * + * Auth: same as the rest of /api/v1 — handled by the global middleware + * (the router is mounted under the auth-gated apiRouter in app.js). + * + * @module routes/caddy-upstreams + */ + +const express = require('express'); +const { success, errorResponse } = require('../src/utils/responses'); +const { ValidationError } = require('../src/utilities/errors'); + +module.exports = function({ asyncHandler, caddyUpstreamWatcher, healthChecker }) { + const router = express.Router(); + + router.get('/caddy/upstreams', asyncHandler(async (req, res) => { + if (!caddyUpstreamWatcher) { + return errorResponse(res, 'Caddy upstream watcher not initialized', 503); + } + success(res, caddyUpstreamWatcher.snapshot()); + }, 'caddy-upstreams-list')); + + router.get('/caddy/upstreams/incidents', asyncHandler(async (req, res) => { + if (!healthChecker) { + return success(res, { incidents: [] }); + } + // Filter the in-memory incidents array to caddy-upstream-dead entries. + const all = Array.isArray(healthChecker.incidents) ? healthChecker.incidents : []; + const open = all + .filter((i) => i && i.type === 'caddy-upstream-dead' && i.status === 'open') + .map((i) => ({ + id: i.id, + serviceId: i.serviceId, + type: i.type, + message: i.message, + severity: i.severity, + createdAt: i.createdAt, + lastOccurrence: i.lastOccurrence, + occurrences: i.occurrences, + details: i.details + })); + success(res, { incidents: open }); + }, 'caddy-upstreams-incidents')); + + // POST /caddy/upstreams/mute body { host, muted } + // POST /caddy/upstreams/:host/mute body { muted: true } OR query ?muted=true + // Both shapes supported because the dashboard code is small and either is + // ergonomic depending on caller. + const handleMute = asyncHandler(async (req, res) => { + if (!caddyUpstreamWatcher) { + return errorResponse(res, 'Caddy upstream watcher not initialized', 503); + } + const host = req.params.host || req.body?.host; + if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) { + throw new ValidationError('host must be a valid host[:port] string'); + } + // Accept muted as boolean body field OR ?muted=true|false query OR + // a { muted: true|false } JSON body. Default to toggling on bare POST + // without a muted value (this is the "mute it" path). + let muted; + if (typeof req.body?.muted === 'boolean') muted = req.body.muted; + else if (typeof req.query.muted === 'string') muted = req.query.muted === 'true'; + else muted = true; // POST with no body = mute + + const result = caddyUpstreamWatcher.setMuted(host, muted); + success(res, result); + }, 'caddy-upstreams-mute'); + + // Bare /mute with JSON body {host, muted}. Default mutes when muted is + // absent or unparseable; require muted === false explicitly to unmute. + router.post('/caddy/upstreams/mute', asyncHandler(async (req, res) => { + if (!caddyUpstreamWatcher) { + return errorResponse(res, 'Caddy upstream watcher not initialized', 503); + } + const { host, muted } = req.body || {}; + if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) { + throw new ValidationError('host must be a valid host[:port] string'); + } + // Explicit boolean coercion — string 'false' should NOT mute. + const wantMuted = muted === undefined ? true : muted === true; + if (caddyUpstreamWatcher.upstreams && !caddyUpstreamWatcher.upstreams.has(host)) { + throw new ValidationError(`host ${host} is not a known upstream (run scan first)`); + } + const result = caddyUpstreamWatcher.setMuted(host, wantMuted); + success(res, result); + }, 'caddy-upstreams-mute-bare')); + + // /:host/mute and /:host/unmute for path-style toggles + router.post('/caddy/upstreams/:host/mute', handleMute); + router.post('/caddy/upstreams/:host/unmute', asyncHandler(async (req, res) => { + if (!caddyUpstreamWatcher) { + return errorResponse(res, 'Caddy upstream watcher not initialized', 503); + } + const host = req.params.host; + if (!host || !/^[a-z0-9._:-]+$/i.test(host)) { + throw new ValidationError('host must be a valid host[:port] string'); + } + const result = caddyUpstreamWatcher.setMuted(host, false); + success(res, result); + }, 'caddy-upstreams-unmute')); + + return router; +}; \ No newline at end of file diff --git a/dashcaddy-api/src/app.js b/dashcaddy-api/src/app.js index 0d786ac..460e464 100644 --- a/dashcaddy-api/src/app.js +++ b/dashcaddy-api/src/app.js @@ -103,6 +103,7 @@ const diskSettingsRoutes = require('../routes/disk-settings'); const aiIntentRoutes = require('../routes/ai-intent'); const logInsightsRoutes = require('../routes/log-insights'); const billingRoutes = require('../routes/billing'); +const caddyUpstreamRoutes = require('../routes/caddy-upstreams'); const DependencyManager = require('./managers/dependency-manager'); const autoRestartRoutes = require('../routes/auto-restart'); const configDriftRoutes = require('../routes/config-drift'); @@ -112,6 +113,7 @@ const { AutoRestartManager } = require('./managers/auto-restart-manager'); const { ConfigDriftDetector } = require('./managers/config-drift-detector'); const SSLMonitor = require('./monitoring/ssl-monitor'); const { DiskSpaceMonitor } = require('./monitoring/disk-space-monitor'); +const caddyUpstreamWatcher = require('./monitoring/caddy-upstream-watcher'); const DNSPropagationChecker = require('./dns/dns-propagation'); // Constants @@ -480,6 +482,15 @@ async function createApp() { diskSpaceMonitor.start(600000); // 10 min log.info('app', 'Disk space monitor initialized', { budgetGB: diskSpaceMonitor.getConfig().diskBudgetGB }); + // Initialize caddy upstream watcher — independent probes of every + // reverse_proxy directive in /etc/caddy/sites/, emits 'dead' incidents + // after 5min of consecutive failures (so a single blip doesn't page). + caddyUpstreamWatcher.log = log; + caddyUpstreamWatcher.healthChecker = healthChecker; + caddyUpstreamWatcher.start(); + ctx.caddyUpstreamWatcher = caddyUpstreamWatcher; + log.info('app', 'Caddy upstream watcher initialized'); + // Initialize DNS propagation checker const dnsPropagationChecker = new DNSPropagationChecker(ctx); ctx.dnsPropagationChecker = dnsPropagationChecker; @@ -794,6 +805,11 @@ async function createApp() { asyncHandler: ctx.asyncHandler, logError: ctx.logError, })); + apiRouter.use(caddyUpstreamRoutes({ + caddyUpstreamWatcher: ctx.caddyUpstreamWatcher, + healthChecker: ctx.healthChecker, + asyncHandler: ctx.asyncHandler, + })); apiRouter.use('/disk', diskSpaceRoutes({ diskSpaceMonitor: ctx.diskSpaceMonitor, asyncHandler: ctx.asyncHandler, diff --git a/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js b/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js new file mode 100644 index 0000000..93ffc4c --- /dev/null +++ b/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js @@ -0,0 +1,444 @@ +/** + * Caddy upstream watcher + * + * Watches every `reverse_proxy ` directive in /etc/caddy/sites/* and + * independently probes each upstream every 60s. After 5 minutes of + * consecutive failures, emits a `caddy-upstream-dead` incident via the shared + * healthChecker so the dashboard can surface it. + * + * This is intentionally separate from Caddy's own `reverse_proxy` health + * checker: Caddy probes log every failure to syslog (the noisy spam the + * dashboard currently sees for `100.120.159.34:5000`), but Caddy never + * surfaces the result to the dashboard or to the API. This watcher gives + * the operator (a) a deduped view, (b) a 5-minute confirmation window so a + * one-off blip doesn't page, and (c) a mute toggle to silence known-dead + * upstreams without editing the Caddyfile. + * + * State persisted to /caddy-upstreams.json. Mute list is part of + * the same file so atomic-write semantics keep state + mutes consistent. + * + * The probe DOES NOT use Caddy's health_uri (that's Caddy's own probe and + * the source of the spam). The probe also stamps `X-DashCaddy-HealthCheck: 1` + * so the `dashcaddy_auth` forward_auth gate on *.sami bypasses for probes + * (same trick as src/monitoring/health-checker.js _doRequest). + * + * @module caddy-upstream-watcher + */ + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const http = require('http'); +const EventEmitter = require('events'); +const platformPaths = require('../../platform-paths'); + +/** Default probe interval: 60s. Independent of Caddy's 10s internal probe. */ +const PROBE_INTERVAL_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_INTERVAL_MS || '60000', 10); + +/** Per-probe timeout. Short — these are liveness pings, not full requests. */ +const PROBE_TIMEOUT_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_TIMEOUT_MS || '5000', 10); + +/** After this many ms of continuous failure, emit a "dead" incident. */ +const DEAD_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_DEAD_AFTER_MS || (5 * 60 * 1000), 10); + +/** After this many ms of continuous success, auto-resolve any open incident. */ +const RESOLVED_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_RESOLVED_AFTER_MS || (60 * 1000), 10); + +/** Status codes that prove the upstream answered. 4xx auth-walled counts as up. */ +const HEALTHY_CODES = new Set([200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429]); + +const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE + || path.join(platformPaths.dataDir || path.dirname(platformPaths.configFile || '.'), 'caddy-upstreams.json'); + +const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites'; + +class CaddyUpstreamWatcher extends EventEmitter { + constructor(opts = {}) { + super(); + this.log = opts.log || console; + this.healthChecker = opts.healthChecker || null; + /** Map keyed by host (host[:port]) */ + this.upstreams = new Map(); + /** Set hosts the user has muted */ + this.muted = new Set(); + /** Set incident IDs currently open — prevents duplicate incidents */ + this.openIncidents = new Set(); + this.timer = null; + this.checking = false; + this.scanTimer = null; + this._loadState(); + } + + /** Begin watching. Idempotent — safe to call twice. */ + start() { + if (this.checking) return; + this.checking = true; + // Initial scan + probe so the dashboard has data immediately after boot. + this.scanSites().catch((e) => this.log.warn('caddy-upstream-watcher', e?.message || String(e))); + this.timer = setInterval(() => this._tick().catch(() => {}), PROBE_INTERVAL_MS); + // Re-scan sites every 5 min so newly added sites get picked up. + this.scanTimer = setInterval(() => this.scanSites().catch(() => {}), 5 * 60 * 1000); + this.log.info?.('caddy-upstream-watcher', 'started', { + probeIntervalMs: PROBE_INTERVAL_MS, + deadAfterMs: DEAD_AFTER_MS, + stateFile: STATE_FILE, + sitesDir: SITES_DIR + }) ?? this.log.info?.('caddy-upstream-watcher', 'started'); + } + + stop() { + if (!this.checking) return; + this.checking = false; + if (this.timer) clearInterval(this.timer); + if (this.scanTimer) clearInterval(this.scanTimer); + this.timer = null; + this.scanTimer = null; + } + + /** Parse /etc/caddy/sites/* and seed/refresh the upstream map. */ + async scanSites() { + let entries; + try { + entries = fs.readdirSync(SITES_DIR); + } catch (e) { + // Sites dir might not exist in dev — that's OK, just skip. + this.log.warn?.('caddy-upstream-watcher', `cannot read ${SITES_DIR}: ${e.message}`); + return; + } + + const seen = new Set(); + for (const entry of entries) { + // Caddy `import` sites have a wild mix of extensions: `.sami`, + // `.caddy`, `.conf` — and ALSO bare hostnames like + // `zap.sami-ahmed.net`, `samitest.space`, `blocks.cryptographic-triangles.org` + // where the "extension" is `.net`/`.space`/`.org`. Filter out known + // non-site junk (readmes, .bak) and accept everything else; the + // reverse_proxy parse below is the real validation. + if (/^README|\.bak$|\.swp$|^\.|^#/.test(entry)) continue; + if (entry === 'Caddyfile' || entry === 'caddyfile') continue; + const filePath = path.join(SITES_DIR, entry); + let content; + try { + content = fs.readFileSync(filePath, 'utf8'); + } catch (_) { continue; } + + // Cheap pre-check: skip files with no reverse_proxy and no brace block + // (README files, .gitignore, etc.). The reverse_proxy regex below is + // the authoritative parse, but this avoids regex-scanning every + // unrelated file in the directory. + if (!/reverse_proxy/i.test(content)) continue; + + // Capture the site block host from the first line: e.g. "arch.sami {" + const siteMatch = content.match(/^\s*([a-z0-9._-]+)\s*\{/im); + const siteName = siteMatch ? siteMatch[1] : entry.replace(/\.(sami|caddy|conf)$/i, ''); + + // Find every reverse_proxy directive. Match common shapes: + // reverse_proxy 100.120.159.34:5000 { ... } + // reverse_proxy http://100.120.159.34:5000 { ... } + // reverse_proxy 100.120.159.34:5000 + const re = /reverse_proxy\s+(?:https?:\/\/)?([0-9]{1,3}(?:\.[0-9]{1,3}){3}|[a-z0-9._-]+)(?::(\d+))?/gi; + let m; + while ((m = re.exec(content)) !== null) { + const host = m[1]; + let port = m[2]; + if (!port) { + if (m[0].includes('https')) port = '443'; + else if (m[0].includes('http://')) port = '80'; + else port = ''; + } + const key = port ? `${host}:${port}` : host; + seen.add(key); + if (!this.upstreams.has(key)) { + this.upstreams.set(key, { + host: key, + ip: host, + port: port || null, + site: siteName, + siteFile: entry, + consecutiveFailures: 0, + lastFailureAt: null, + lastSuccessAt: null, + lastError: null, + lastCheckedAt: null, + status: 'unknown' + }); + } else { + // Refresh site name/file in case the file was renamed. + const u = this.upstreams.get(key); + u.site = siteName; + u.siteFile = entry; + } + } + } + + // Drop upstreams that disappeared from the Caddyfile (removed/renamed site). + for (const key of Array.from(this.upstreams.keys())) { + if (!seen.has(key)) this.upstreams.delete(key); + } + + this._saveState(); + } + + /** Single probe tick over every upstream. */ + async _tick() { + const probes = []; + for (const u of this.upstreams.values()) { + if (this.muted.has(u.host)) continue; + probes.push(this._probeOne(u).catch((e) => { + this.log.warn?.('caddy-upstream-watcher', `probe failed for ${u.host}: ${e.message}`); + })); + } + await Promise.all(probes); + this._saveState(); + this.emit('tick', this.snapshot()); + } + + /** Probe a single upstream and update state. */ + async _probeOne(u) { + const result = await this._doProbe(u.ip, u.port); + u.lastCheckedAt = new Date().toISOString(); + + if (result.healthy) { + u.consecutiveFailures = 0; + u.lastSuccessAt = u.lastCheckedAt; + u.lastError = null; + // Resolve open incident if upstream is healthy for RESOLVED_AFTER_MS. + this._maybeResolve(u); + // Only flip to 'up' if the upstream has been healthy long enough to not + // be a flapping signal — short blips are normal and we want the dashboard + // to be stable. After one full successful check we mark 'up' but the + // incident resolution waits for RESOLVED_AFTER_MS. + u.status = 'up'; + } else { + u.consecutiveFailures += 1; + u.lastFailureAt = u.lastCheckedAt; + u.lastError = result.error || `HTTP ${result.statusCode || 'unknown'}`; + // First failure flips status to 'down' immediately for the dashboard, but + // we only OPEN an incident after the upstream has been continuously failing + // for DEAD_AFTER_MS (5 min by default) so a single transient blip doesn't + // page anyone. + u.status = 'down'; + this._maybeOpenIncident(u); + } + } + + _maybeOpenIncident(u) { + if (!this.healthChecker) return; + // "failingForMs" = continuous time the upstream has been unhealthy. + // Use lastSuccessAt as the anchor — if it was up 7min ago and is still + // down now, that's 7 minutes of continuous failure regardless of how many + // individual probe failures have piled up in between. Falls back to + // consecutiveFailures * interval when there's no success anchor (e.g. we've + // never seen the upstream healthy since startup). + const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null; + const failingForMs = lastSuccessMs !== null + ? Math.max(0, Date.now() - lastSuccessMs) + : u.consecutiveFailures * PROBE_INTERVAL_MS; + if (failingForMs < DEAD_AFTER_MS) return; + if (this.openIncidents.has(u.host)) return; + + // Mimic the shape HealthChecker.createIncident expects. + try { + this.healthChecker.createIncident(u.host, 'caddy-upstream-dead', + `Caddy upstream ${u.host} (site ${u.site}) unreachable for ${Math.round(failingForMs / 60000)}m: ${u.lastError || 'no response'}`, + { + serviceId: u.host, + timestamp: u.lastFailureAt, + status: 'down', + error: u.lastError, + details: { site: u.site, siteFile: u.siteFile } + } + ); + this.openIncidents.add(u.host); + this.emit('upstream-dead', u); + this.log.warn?.('caddy-upstream-watcher', `upstream dead: ${u.host} (${u.site})`); + } catch (e) { + this.log.warn?.('caddy-upstream-watcher', `incident create failed: ${e.message}`); + } + } + + _maybeResolve(u) { + if (!this.healthChecker) return; + if (!this.openIncidents.has(u.host)) return; + const downSince = u.lastFailureAt ? new Date(u.lastFailureAt).getTime() : 0; + const recoveredForMs = downSince ? Date.now() - downSince : 0; + if (recoveredForMs < RESOLVED_AFTER_MS) return; + try { + this.healthChecker.resolveIncident(u.host, 'caddy-upstream-dead', { + serviceId: u.host, + timestamp: u.lastSuccessAt || new Date().toISOString(), + status: 'up' + }); + this.openIncidents.delete(u.host); + this.emit('upstream-recovered', u); + this.log.info?.('caddy-upstream-watcher', `upstream recovered: ${u.host}`); + } catch (e) { + this.log.warn?.('caddy-upstream-watcher', `incident resolve failed: ${e.message}`); + } + } + + _doProbe(host, port) { + return new Promise((resolve) => { + const isHttps = port === '443'; + const lib = isHttps ? https : http; + const opts = { + hostname: host, + port: port || (isHttps ? 443 : 80), + method: 'HEAD', + path: '/', + timeout: PROBE_TIMEOUT_MS, + headers: { 'X-DashCaddy-HealthCheck': '1', 'User-Agent': 'DashCaddy-CaddyUpstreamWatcher/1' }, + rejectUnauthorized: false + }; + const req = lib.request(opts, (res) => { + res.resume(); + const healthy = HEALTHY_CODES.has(res.statusCode); + resolve({ healthy, statusCode: res.statusCode }); + }); + req.on('timeout', () => { + req.destroy(new Error('probe timeout')); + }); + req.on('error', (err) => { + resolve({ healthy: false, error: err.message }); + }); + req.end(); + }); + } + + /** Public snapshot for the API/UI. */ + snapshot() { + const list = []; + for (const u of this.upstreams.values()) { + const muted = this.muted.has(u.host); + // Same anchor as _maybeOpenIncident: time since the last successful + // probe. If we've never seen a success, fall back to consecutive + // failures × probe interval as a worst-case lower bound. + const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null; + let failingFor = 0; + if (!muted) { + if (lastSuccessMs !== null) { + failingFor = Math.max(0, Date.now() - lastSuccessMs); + } else if (u.status === 'down') { + failingFor = u.consecutiveFailures * PROBE_INTERVAL_MS; + } + } + list.push({ + host: u.host, + site: u.site, + siteFile: u.siteFile, + status: muted ? 'muted' : u.status, + consecutiveFailures: u.consecutiveFailures, + lastCheckedAt: u.lastCheckedAt, + lastSuccessAt: u.lastSuccessAt, + lastFailureAt: u.lastFailureAt, + lastError: u.lastError, + failingForMs: failingFor, + muted, + dead: !muted && failingFor >= DEAD_AFTER_MS + }); + } + // Sort: dead first, then down, then up, then unknown. Within each, by host. + list.sort((a, b) => { + const order = { dead: 0, down: 1, muted: 2, up: 3, unknown: 4 }; + const oa = order[a.dead ? 'dead' : a.status] ?? 9; + const ob = order[b.dead ? 'dead' : b.status] ?? 9; + if (oa !== ob) return oa - ob; + return a.host.localeCompare(b.host); + }); + return { + upstreams: list, + config: { + probeIntervalMs: PROBE_INTERVAL_MS, + deadAfterMs: DEAD_AFTER_MS, + resolvedAfterMs: RESOLVED_AFTER_MS, + sitesDir: SITES_DIR + } + }; + } + + setMuted(host, muted) { + if (muted) { + this.muted.add(host); + } else { + this.muted.delete(host); + // Reset failure state on unmute so we don't immediately re-incident a + // upstream that just came off mute. + const u = this.upstreams.get(host); + if (u) { + u.consecutiveFailures = 0; + u.lastError = null; + u.lastFailureAt = null; + u.status = 'unknown'; + } + } + this._saveState(); + return { host, muted: !!muted }; + } + + isMuted(host) { return this.muted.has(host); } + + _loadState() { + try { + if (!fs.existsSync(STATE_FILE)) return; + const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')); + if (Array.isArray(data.muted)) this.muted = new Set(data.muted); + // Don't reload upstreams from disk — sites dir is the source of truth. + // But preserve last-check state for hosts that still exist. + if (data.upstreams && typeof data.upstreams === 'object') { + this._restoreUpstreamStates(data.upstreams); + } + } catch (e) { + this.log.warn?.('caddy-upstream-watcher', `state load failed: ${e.message}`); + } + } + + _restoreUpstreamStates(persisted) { + for (const [host, st] of Object.entries(persisted)) { + if (this.upstreams.has(host)) continue; + this.upstreams.set(host, { + host, + ip: st.ip || host.split(':')[0], + port: st.port || null, + site: st.site || '', + siteFile: st.siteFile || '', + consecutiveFailures: st.consecutiveFailures || 0, + lastFailureAt: st.lastFailureAt || null, + lastSuccessAt: st.lastSuccessAt || null, + lastError: st.lastError || null, + lastCheckedAt: st.lastCheckedAt || null, + status: 'unknown' + }); + } + } + + _saveState() { + try { + const dir = path.dirname(STATE_FILE); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + const upstreams = {}; + for (const [k, v] of this.upstreams.entries()) { + upstreams[k] = { + ip: v.ip, + port: v.port, + site: v.site, + siteFile: v.siteFile, + consecutiveFailures: v.consecutiveFailures, + lastFailureAt: v.lastFailureAt, + lastSuccessAt: v.lastSuccessAt, + lastError: v.lastError, + lastCheckedAt: v.lastCheckedAt + }; + } + const tmp = STATE_FILE + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify({ muted: Array.from(this.muted), upstreams }, null, 2)); + fs.renameSync(tmp, STATE_FILE); + } catch (e) { + this.log.warn?.('caddy-upstream-watcher', `state save failed: ${e.message}`); + } + } +} + +// Singleton — matches the pattern of health-checker.js so it integrates +// without a separate instantiation site. +module.exports = new CaddyUpstreamWatcher(); +module.exports.CaddyUpstreamWatcher = CaddyUpstreamWatcher; \ No newline at end of file