feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]
CI / Test & Lint (push) Canceled after 0s
CI / Security audit (push) Canceled after 0s

Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:

- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
  reverse_proxy directive, probes each upstream every 60s independent of
  Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
  failures via the existing healthChecker. Mute list persisted to
  data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
  forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
  GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
  POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
  POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
  All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.

GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:

1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
   silently skipped real prod filenames like zap.sami-ahmed.net,
   samitest.space, blocks.cryptographic-triangles.org where the file
   extension is .net/.space/.org. Replaced with positive filter that
   excludes README/.bak/.swp/Caddyfile + content pre-check
   (must contain 'reverse_proxy'). Added test covering the prod filenames.

2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
   MUTED the host because the bare route used `muted !== false` which is
   true for the string 'false'. Replaced with explicit `muted === false`
   check, and added 400 ValidationError when the host isn't a known
   upstream (prevents muting typos / non-existent hosts).

Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.
This commit is contained in:
Sami Ahmed
2026-08-17 17:11:27 -07:00
parent 6d875e4631
commit 45cfa83bad
5 changed files with 1090 additions and 0 deletions
+109
View File
@@ -0,0 +1,109 @@
/**
* Caddy upstreams routes
*
* Exposes:
* GET /api/v1/caddy/upstreams — full snapshot
* GET /api/v1/caddy/upstreams/incidents — open dead-upstream incidents (via healthChecker)
* POST /api/v1/caddy/upstreams/:host/mute — body { muted: true|false } (also via query ?muted=true)
*
* Auth: same as the rest of /api/v1 — handled by the global middleware
* (the router is mounted under the auth-gated apiRouter in app.js).
*
* @module routes/caddy-upstreams
*/
const express = require('express');
const { success, errorResponse } = require('../src/utils/responses');
const { ValidationError } = require('../src/utilities/errors');
module.exports = function({ asyncHandler, caddyUpstreamWatcher, healthChecker }) {
const router = express.Router();
router.get('/caddy/upstreams', asyncHandler(async (req, res) => {
if (!caddyUpstreamWatcher) {
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
}
success(res, caddyUpstreamWatcher.snapshot());
}, 'caddy-upstreams-list'));
router.get('/caddy/upstreams/incidents', asyncHandler(async (req, res) => {
if (!healthChecker) {
return success(res, { incidents: [] });
}
// Filter the in-memory incidents array to caddy-upstream-dead entries.
const all = Array.isArray(healthChecker.incidents) ? healthChecker.incidents : [];
const open = all
.filter((i) => i && i.type === 'caddy-upstream-dead' && i.status === 'open')
.map((i) => ({
id: i.id,
serviceId: i.serviceId,
type: i.type,
message: i.message,
severity: i.severity,
createdAt: i.createdAt,
lastOccurrence: i.lastOccurrence,
occurrences: i.occurrences,
details: i.details
}));
success(res, { incidents: open });
}, 'caddy-upstreams-incidents'));
// POST /caddy/upstreams/mute body { host, muted }
// POST /caddy/upstreams/:host/mute body { muted: true } OR query ?muted=true
// Both shapes supported because the dashboard code is small and either is
// ergonomic depending on caller.
const handleMute = asyncHandler(async (req, res) => {
if (!caddyUpstreamWatcher) {
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
}
const host = req.params.host || req.body?.host;
if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
throw new ValidationError('host must be a valid host[:port] string');
}
// Accept muted as boolean body field OR ?muted=true|false query OR
// a { muted: true|false } JSON body. Default to toggling on bare POST
// without a muted value (this is the "mute it" path).
let muted;
if (typeof req.body?.muted === 'boolean') muted = req.body.muted;
else if (typeof req.query.muted === 'string') muted = req.query.muted === 'true';
else muted = true; // POST with no body = mute
const result = caddyUpstreamWatcher.setMuted(host, muted);
success(res, result);
}, 'caddy-upstreams-mute');
// Bare /mute with JSON body {host, muted}. Default mutes when muted is
// absent or unparseable; require muted === false explicitly to unmute.
router.post('/caddy/upstreams/mute', asyncHandler(async (req, res) => {
if (!caddyUpstreamWatcher) {
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
}
const { host, muted } = req.body || {};
if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
throw new ValidationError('host must be a valid host[:port] string');
}
// Explicit boolean coercion — string 'false' should NOT mute.
const wantMuted = muted === undefined ? true : muted === true;
if (caddyUpstreamWatcher.upstreams && !caddyUpstreamWatcher.upstreams.has(host)) {
throw new ValidationError(`host ${host} is not a known upstream (run scan first)`);
}
const result = caddyUpstreamWatcher.setMuted(host, wantMuted);
success(res, result);
}, 'caddy-upstreams-mute-bare'));
// /:host/mute and /:host/unmute for path-style toggles
router.post('/caddy/upstreams/:host/mute', handleMute);
router.post('/caddy/upstreams/:host/unmute', asyncHandler(async (req, res) => {
if (!caddyUpstreamWatcher) {
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
}
const host = req.params.host;
if (!host || !/^[a-z0-9._:-]+$/i.test(host)) {
throw new ValidationError('host must be a valid host[:port] string');
}
const result = caddyUpstreamWatcher.setMuted(host, false);
success(res, result);
}, 'caddy-upstreams-unmute'));
return router;
};