Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:
- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
reverse_proxy directive, probes each upstream every 60s independent of
Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
failures via the existing healthChecker. Mute list persisted to
data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.
GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:
1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
silently skipped real prod filenames like zap.sami-ahmed.net,
samitest.space, blocks.cryptographic-triangles.org where the file
extension is .net/.space/.org. Replaced with positive filter that
excludes README/.bak/.swp/Caddyfile + content pre-check
(must contain 'reverse_proxy'). Added test covering the prod filenames.
2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
MUTED the host because the bare route used `muted !== false` which is
true for the string 'false'. Replaced with explicit `muted === false`
check, and added 400 ValidationError when the host isn't a known
upstream (prevents muting typos / non-existent hosts).
Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.
109 lines
4.6 KiB
JavaScript
109 lines
4.6 KiB
JavaScript
/**
|
|
* Caddy upstreams routes
|
|
*
|
|
* Exposes:
|
|
* GET /api/v1/caddy/upstreams — full snapshot
|
|
* GET /api/v1/caddy/upstreams/incidents — open dead-upstream incidents (via healthChecker)
|
|
* POST /api/v1/caddy/upstreams/:host/mute — body { muted: true|false } (also via query ?muted=true)
|
|
*
|
|
* Auth: same as the rest of /api/v1 — handled by the global middleware
|
|
* (the router is mounted under the auth-gated apiRouter in app.js).
|
|
*
|
|
* @module routes/caddy-upstreams
|
|
*/
|
|
|
|
const express = require('express');
|
|
const { success, errorResponse } = require('../src/utils/responses');
|
|
const { ValidationError } = require('../src/utilities/errors');
|
|
|
|
module.exports = function({ asyncHandler, caddyUpstreamWatcher, healthChecker }) {
|
|
const router = express.Router();
|
|
|
|
router.get('/caddy/upstreams', asyncHandler(async (req, res) => {
|
|
if (!caddyUpstreamWatcher) {
|
|
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
|
|
}
|
|
success(res, caddyUpstreamWatcher.snapshot());
|
|
}, 'caddy-upstreams-list'));
|
|
|
|
router.get('/caddy/upstreams/incidents', asyncHandler(async (req, res) => {
|
|
if (!healthChecker) {
|
|
return success(res, { incidents: [] });
|
|
}
|
|
// Filter the in-memory incidents array to caddy-upstream-dead entries.
|
|
const all = Array.isArray(healthChecker.incidents) ? healthChecker.incidents : [];
|
|
const open = all
|
|
.filter((i) => i && i.type === 'caddy-upstream-dead' && i.status === 'open')
|
|
.map((i) => ({
|
|
id: i.id,
|
|
serviceId: i.serviceId,
|
|
type: i.type,
|
|
message: i.message,
|
|
severity: i.severity,
|
|
createdAt: i.createdAt,
|
|
lastOccurrence: i.lastOccurrence,
|
|
occurrences: i.occurrences,
|
|
details: i.details
|
|
}));
|
|
success(res, { incidents: open });
|
|
}, 'caddy-upstreams-incidents'));
|
|
|
|
// POST /caddy/upstreams/mute body { host, muted }
|
|
// POST /caddy/upstreams/:host/mute body { muted: true } OR query ?muted=true
|
|
// Both shapes supported because the dashboard code is small and either is
|
|
// ergonomic depending on caller.
|
|
const handleMute = asyncHandler(async (req, res) => {
|
|
if (!caddyUpstreamWatcher) {
|
|
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
|
|
}
|
|
const host = req.params.host || req.body?.host;
|
|
if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
|
|
throw new ValidationError('host must be a valid host[:port] string');
|
|
}
|
|
// Accept muted as boolean body field OR ?muted=true|false query OR
|
|
// a { muted: true|false } JSON body. Default to toggling on bare POST
|
|
// without a muted value (this is the "mute it" path).
|
|
let muted;
|
|
if (typeof req.body?.muted === 'boolean') muted = req.body.muted;
|
|
else if (typeof req.query.muted === 'string') muted = req.query.muted === 'true';
|
|
else muted = true; // POST with no body = mute
|
|
|
|
const result = caddyUpstreamWatcher.setMuted(host, muted);
|
|
success(res, result);
|
|
}, 'caddy-upstreams-mute');
|
|
|
|
// Bare /mute with JSON body {host, muted}. Default mutes when muted is
|
|
// absent or unparseable; require muted === false explicitly to unmute.
|
|
router.post('/caddy/upstreams/mute', asyncHandler(async (req, res) => {
|
|
if (!caddyUpstreamWatcher) {
|
|
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
|
|
}
|
|
const { host, muted } = req.body || {};
|
|
if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
|
|
throw new ValidationError('host must be a valid host[:port] string');
|
|
}
|
|
// Explicit boolean coercion — string 'false' should NOT mute.
|
|
const wantMuted = muted === undefined ? true : muted === true;
|
|
if (caddyUpstreamWatcher.upstreams && !caddyUpstreamWatcher.upstreams.has(host)) {
|
|
throw new ValidationError(`host ${host} is not a known upstream (run scan first)`);
|
|
}
|
|
const result = caddyUpstreamWatcher.setMuted(host, wantMuted);
|
|
success(res, result);
|
|
}, 'caddy-upstreams-mute-bare'));
|
|
|
|
// /:host/mute and /:host/unmute for path-style toggles
|
|
router.post('/caddy/upstreams/:host/mute', handleMute);
|
|
router.post('/caddy/upstreams/:host/unmute', asyncHandler(async (req, res) => {
|
|
if (!caddyUpstreamWatcher) {
|
|
return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
|
|
}
|
|
const host = req.params.host;
|
|
if (!host || !/^[a-z0-9._:-]+$/i.test(host)) {
|
|
throw new ValidationError('host must be a valid host[:port] string');
|
|
}
|
|
const result = caddyUpstreamWatcher.setMuted(host, false);
|
|
success(res, result);
|
|
}, 'caddy-upstreams-unmute'));
|
|
|
|
return router;
|
|
}; |