feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]
Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:
- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
reverse_proxy directive, probes each upstream every 60s independent of
Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
failures via the existing healthChecker. Mute list persisted to
data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.
GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:
1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
silently skipped real prod filenames like zap.sami-ahmed.net,
samitest.space, blocks.cryptographic-triangles.org where the file
extension is .net/.space/.org. Replaced with positive filter that
excludes README/.bak/.swp/Caddyfile + content pre-check
(must contain 'reverse_proxy'). Added test covering the prod filenames.
2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
MUTED the host because the bare route used `muted !== false` which is
true for the string 'false'. Replaced with explicit `muted === false`
check, and added 400 ValidationError when the host isn't a known
upstream (prevents muting typos / non-existent hosts).
Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.
This commit is contained in:
@@ -103,6 +103,7 @@ const diskSettingsRoutes = require('../routes/disk-settings');
|
||||
const aiIntentRoutes = require('../routes/ai-intent');
|
||||
const logInsightsRoutes = require('../routes/log-insights');
|
||||
const billingRoutes = require('../routes/billing');
|
||||
const caddyUpstreamRoutes = require('../routes/caddy-upstreams');
|
||||
const DependencyManager = require('./managers/dependency-manager');
|
||||
const autoRestartRoutes = require('../routes/auto-restart');
|
||||
const configDriftRoutes = require('../routes/config-drift');
|
||||
@@ -112,6 +113,7 @@ const { AutoRestartManager } = require('./managers/auto-restart-manager');
|
||||
const { ConfigDriftDetector } = require('./managers/config-drift-detector');
|
||||
const SSLMonitor = require('./monitoring/ssl-monitor');
|
||||
const { DiskSpaceMonitor } = require('./monitoring/disk-space-monitor');
|
||||
const caddyUpstreamWatcher = require('./monitoring/caddy-upstream-watcher');
|
||||
const DNSPropagationChecker = require('./dns/dns-propagation');
|
||||
|
||||
// Constants
|
||||
@@ -480,6 +482,15 @@ async function createApp() {
|
||||
diskSpaceMonitor.start(600000); // 10 min
|
||||
log.info('app', 'Disk space monitor initialized', { budgetGB: diskSpaceMonitor.getConfig().diskBudgetGB });
|
||||
|
||||
// Initialize caddy upstream watcher — independent probes of every
|
||||
// reverse_proxy directive in /etc/caddy/sites/, emits 'dead' incidents
|
||||
// after 5min of consecutive failures (so a single blip doesn't page).
|
||||
caddyUpstreamWatcher.log = log;
|
||||
caddyUpstreamWatcher.healthChecker = healthChecker;
|
||||
caddyUpstreamWatcher.start();
|
||||
ctx.caddyUpstreamWatcher = caddyUpstreamWatcher;
|
||||
log.info('app', 'Caddy upstream watcher initialized');
|
||||
|
||||
// Initialize DNS propagation checker
|
||||
const dnsPropagationChecker = new DNSPropagationChecker(ctx);
|
||||
ctx.dnsPropagationChecker = dnsPropagationChecker;
|
||||
@@ -794,6 +805,11 @@ async function createApp() {
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
logError: ctx.logError,
|
||||
}));
|
||||
apiRouter.use(caddyUpstreamRoutes({
|
||||
caddyUpstreamWatcher: ctx.caddyUpstreamWatcher,
|
||||
healthChecker: ctx.healthChecker,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
}));
|
||||
apiRouter.use('/disk', diskSpaceRoutes({
|
||||
diskSpaceMonitor: ctx.diskSpaceMonitor,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
|
||||
Reference in New Issue
Block a user