feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]

Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:

- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
  reverse_proxy directive, probes each upstream every 60s independent of
  Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
  failures via the existing healthChecker. Mute list persisted to
  data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
  forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
  GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
  POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
  POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
  All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.

GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:

1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
   silently skipped real prod filenames like zap.sami-ahmed.net,
   samitest.space, blocks.cryptographic-triangles.org where the file
   extension is .net/.space/.org. Replaced with positive filter that
   excludes README/.bak/.swp/Caddyfile + content pre-check
   (must contain 'reverse_proxy'). Added test covering the prod filenames.

2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
   MUTED the host because the bare route used `muted !== false` which is
   true for the string 'false'. Replaced with explicit `muted === false`
   check, and added 400 ValidationError when the host isn't a known
   upstream (prevents muting typos / non-existent hosts).

Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.

dashcaddy-api/src/app.js

@@ -103,6 +103,7 @@ const diskSettingsRoutes = require('../routes/disk-settings');
 const aiIntentRoutes = require('../routes/ai-intent');
 const logInsightsRoutes = require('../routes/log-insights');
 const billingRoutes = require('../routes/billing');
+const caddyUpstreamRoutes = require('../routes/caddy-upstreams');
 const DependencyManager = require('./managers/dependency-manager');
 const autoRestartRoutes = require('../routes/auto-restart');
 const configDriftRoutes = require('../routes/config-drift');
@@ -112,6 +113,7 @@ const { AutoRestartManager } = require('./managers/auto-restart-manager');
 const { ConfigDriftDetector } = require('./managers/config-drift-detector');
 const SSLMonitor = require('./monitoring/ssl-monitor');
 const { DiskSpaceMonitor } = require('./monitoring/disk-space-monitor');
+const caddyUpstreamWatcher = require('./monitoring/caddy-upstream-watcher');
 const DNSPropagationChecker = require('./dns/dns-propagation');
 
 // Constants
@@ -480,6 +482,15 @@ async function createApp() {
   diskSpaceMonitor.start(600000); // 10 min
   log.info('app', 'Disk space monitor initialized', { budgetGB: diskSpaceMonitor.getConfig().diskBudgetGB });
 
+  // Initialize caddy upstream watcher — independent probes of every
+  // reverse_proxy directive in /etc/caddy/sites/, emits 'dead' incidents
+  // after 5min of consecutive failures (so a single blip doesn't page).
+  caddyUpstreamWatcher.log = log;
+  caddyUpstreamWatcher.healthChecker = healthChecker;
+  caddyUpstreamWatcher.start();
+  ctx.caddyUpstreamWatcher = caddyUpstreamWatcher;
+  log.info('app', 'Caddy upstream watcher initialized');
+
   // Initialize DNS propagation checker
   const dnsPropagationChecker = new DNSPropagationChecker(ctx);
   ctx.dnsPropagationChecker = dnsPropagationChecker;
@@ -794,6 +805,11 @@ async function createApp() {
     asyncHandler: ctx.asyncHandler,
     logError: ctx.logError,
   }));
+  apiRouter.use(caddyUpstreamRoutes({
+    caddyUpstreamWatcher: ctx.caddyUpstreamWatcher,
+    healthChecker: ctx.healthChecker,
+    asyncHandler: ctx.asyncHandler,
+  }));
   apiRouter.use('/disk', diskSpaceRoutes({
     diskSpaceMonitor: ctx.diskSpaceMonitor,
     asyncHandler: ctx.asyncHandler,

dashcaddy-api/src/monitoring/caddy-upstream-watcher.js

@@ -0,0 +1,444 @@
+/**
+ * Caddy upstream watcher
+ *
+ * Watches every `reverse_proxy <host>` directive in /etc/caddy/sites/* and
+ * independently probes each upstream every 60s. After 5 minutes of
+ * consecutive failures, emits a `caddy-upstream-dead` incident via the shared
+ * healthChecker so the dashboard can surface it.
+ *
+ * This is intentionally separate from Caddy's own `reverse_proxy` health
+ * checker: Caddy probes log every failure to syslog (the noisy spam the
+ * dashboard currently sees for `100.120.159.34:5000`), but Caddy never
+ * surfaces the result to the dashboard or to the API. This watcher gives
+ * the operator (a) a deduped view, (b) a 5-minute confirmation window so a
+ * one-off blip doesn't page, and (c) a mute toggle to silence known-dead
+ * upstreams without editing the Caddyfile.
+ *
+ * State persisted to <dataDir>/caddy-upstreams.json. Mute list is part of
+ * the same file so atomic-write semantics keep state + mutes consistent.
+ *
+ * The probe DOES NOT use Caddy's health_uri (that's Caddy's own probe and
+ * the source of the spam). The probe also stamps `X-DashCaddy-HealthCheck: 1`
+ * so the `dashcaddy_auth` forward_auth gate on *.sami bypasses for probes
+ * (same trick as src/monitoring/health-checker.js _doRequest).
+ *
+ * @module caddy-upstream-watcher
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+const EventEmitter = require('events');
+const platformPaths = require('../../platform-paths');
+
+/** Default probe interval: 60s. Independent of Caddy's 10s internal probe. */
+const PROBE_INTERVAL_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_INTERVAL_MS || '60000', 10);
+
+/** Per-probe timeout. Short — these are liveness pings, not full requests. */
+const PROBE_TIMEOUT_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_TIMEOUT_MS || '5000', 10);
+
+/** After this many ms of continuous failure, emit a "dead" incident. */
+const DEAD_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_DEAD_AFTER_MS || (5 * 60 * 1000), 10);
+
+/** After this many ms of continuous success, auto-resolve any open incident. */
+const RESOLVED_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_RESOLVED_AFTER_MS || (60 * 1000), 10);
+
+/** Status codes that prove the upstream answered. 4xx auth-walled counts as up. */
+const HEALTHY_CODES = new Set([200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429]);
+
+const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE
+  || path.join(platformPaths.dataDir || path.dirname(platformPaths.configFile || '.'), 'caddy-upstreams.json');
+
+const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites';
+
+class CaddyUpstreamWatcher extends EventEmitter {
+  constructor(opts = {}) {
+    super();
+    this.log = opts.log || console;
+    this.healthChecker = opts.healthChecker || null;
+    /** Map<string, UpstreamState> keyed by host (host[:port]) */
+    this.upstreams = new Map();
+    /** Set<string> hosts the user has muted */
+    this.muted = new Set();
+    /** Set<string> incident IDs currently open — prevents duplicate incidents */
+    this.openIncidents = new Set();
+    this.timer = null;
+    this.checking = false;
+    this.scanTimer = null;
+    this._loadState();
+  }
+
+  /** Begin watching. Idempotent — safe to call twice. */
+  start() {
+    if (this.checking) return;
+    this.checking = true;
+    // Initial scan + probe so the dashboard has data immediately after boot.
+    this.scanSites().catch((e) => this.log.warn('caddy-upstream-watcher', e?.message || String(e)));
+    this.timer = setInterval(() => this._tick().catch(() => {}), PROBE_INTERVAL_MS);
+    // Re-scan sites every 5 min so newly added sites get picked up.
+    this.scanTimer = setInterval(() => this.scanSites().catch(() => {}), 5 * 60 * 1000);
+    this.log.info?.('caddy-upstream-watcher', 'started', {
+      probeIntervalMs: PROBE_INTERVAL_MS,
+      deadAfterMs: DEAD_AFTER_MS,
+      stateFile: STATE_FILE,
+      sitesDir: SITES_DIR
+    }) ?? this.log.info?.('caddy-upstream-watcher', 'started');
+  }
+
+  stop() {
+    if (!this.checking) return;
+    this.checking = false;
+    if (this.timer) clearInterval(this.timer);
+    if (this.scanTimer) clearInterval(this.scanTimer);
+    this.timer = null;
+    this.scanTimer = null;
+  }
+
+  /** Parse /etc/caddy/sites/* and seed/refresh the upstream map. */
+  async scanSites() {
+    let entries;
+    try {
+      entries = fs.readdirSync(SITES_DIR);
+    } catch (e) {
+      // Sites dir might not exist in dev — that's OK, just skip.
+      this.log.warn?.('caddy-upstream-watcher', `cannot read ${SITES_DIR}: ${e.message}`);
+      return;
+    }
+
+    const seen = new Set();
+    for (const entry of entries) {
+      // Caddy `import` sites have a wild mix of extensions: `.sami`,
+      // `.caddy`, `.conf` — and ALSO bare hostnames like
+      // `zap.sami-ahmed.net`, `samitest.space`, `blocks.cryptographic-triangles.org`
+      // where the "extension" is `.net`/`.space`/`.org`. Filter out known
+      // non-site junk (readmes, .bak) and accept everything else; the
+      // reverse_proxy parse below is the real validation.
+      if (/^README|\.bak$|\.swp$|^\.|^#/.test(entry)) continue;
+      if (entry === 'Caddyfile' || entry === 'caddyfile') continue;
+      const filePath = path.join(SITES_DIR, entry);
+      let content;
+      try {
+        content = fs.readFileSync(filePath, 'utf8');
+      } catch (_) { continue; }
+
+      // Cheap pre-check: skip files with no reverse_proxy and no brace block
+      // (README files, .gitignore, etc.). The reverse_proxy regex below is
+      // the authoritative parse, but this avoids regex-scanning every
+      // unrelated file in the directory.
+      if (!/reverse_proxy/i.test(content)) continue;
+
+      // Capture the site block host from the first line: e.g. "arch.sami {"
+      const siteMatch = content.match(/^\s*([a-z0-9._-]+)\s*\{/im);
+      const siteName = siteMatch ? siteMatch[1] : entry.replace(/\.(sami|caddy|conf)$/i, '');
+
+      // Find every reverse_proxy <host[:port]> directive. Match common shapes:
+      //   reverse_proxy 100.120.159.34:5000 { ... }
+      //   reverse_proxy http://100.120.159.34:5000 { ... }
+      //   reverse_proxy 100.120.159.34:5000
+      const re = /reverse_proxy\s+(?:https?:\/\/)?([0-9]{1,3}(?:\.[0-9]{1,3}){3}|[a-z0-9._-]+)(?::(\d+))?/gi;
+      let m;
+      while ((m = re.exec(content)) !== null) {
+        const host = m[1];
+        let port = m[2];
+        if (!port) {
+          if (m[0].includes('https')) port = '443';
+          else if (m[0].includes('http://')) port = '80';
+          else port = '';
+        }
+        const key = port ? `${host}:${port}` : host;
+        seen.add(key);
+        if (!this.upstreams.has(key)) {
+          this.upstreams.set(key, {
+            host: key,
+            ip: host,
+            port: port || null,
+            site: siteName,
+            siteFile: entry,
+            consecutiveFailures: 0,
+            lastFailureAt: null,
+            lastSuccessAt: null,
+            lastError: null,
+            lastCheckedAt: null,
+            status: 'unknown'
+          });
+        } else {
+          // Refresh site name/file in case the file was renamed.
+          const u = this.upstreams.get(key);
+          u.site = siteName;
+          u.siteFile = entry;
+        }
+      }
+    }
+
+    // Drop upstreams that disappeared from the Caddyfile (removed/renamed site).
+    for (const key of Array.from(this.upstreams.keys())) {
+      if (!seen.has(key)) this.upstreams.delete(key);
+    }
+
+    this._saveState();
+  }
+
+  /** Single probe tick over every upstream. */
+  async _tick() {
+    const probes = [];
+    for (const u of this.upstreams.values()) {
+      if (this.muted.has(u.host)) continue;
+      probes.push(this._probeOne(u).catch((e) => {
+        this.log.warn?.('caddy-upstream-watcher', `probe failed for ${u.host}: ${e.message}`);
+      }));
+    }
+    await Promise.all(probes);
+    this._saveState();
+    this.emit('tick', this.snapshot());
+  }
+
+  /** Probe a single upstream and update state. */
+  async _probeOne(u) {
+    const result = await this._doProbe(u.ip, u.port);
+    u.lastCheckedAt = new Date().toISOString();
+
+    if (result.healthy) {
+      u.consecutiveFailures = 0;
+      u.lastSuccessAt = u.lastCheckedAt;
+      u.lastError = null;
+      // Resolve open incident if upstream is healthy for RESOLVED_AFTER_MS.
+      this._maybeResolve(u);
+      // Only flip to 'up' if the upstream has been healthy long enough to not
+      // be a flapping signal — short blips are normal and we want the dashboard
+      // to be stable. After one full successful check we mark 'up' but the
+      // incident resolution waits for RESOLVED_AFTER_MS.
+      u.status = 'up';
+    } else {
+      u.consecutiveFailures += 1;
+      u.lastFailureAt = u.lastCheckedAt;
+      u.lastError = result.error || `HTTP ${result.statusCode || 'unknown'}`;
+      // First failure flips status to 'down' immediately for the dashboard, but
+      // we only OPEN an incident after the upstream has been continuously failing
+      // for DEAD_AFTER_MS (5 min by default) so a single transient blip doesn't
+      // page anyone.
+      u.status = 'down';
+      this._maybeOpenIncident(u);
+    }
+  }
+
+  _maybeOpenIncident(u) {
+    if (!this.healthChecker) return;
+    // "failingForMs" = continuous time the upstream has been unhealthy.
+    // Use lastSuccessAt as the anchor — if it was up 7min ago and is still
+    // down now, that's 7 minutes of continuous failure regardless of how many
+    // individual probe failures have piled up in between. Falls back to
+    // consecutiveFailures * interval when there's no success anchor (e.g. we've
+    // never seen the upstream healthy since startup).
+    const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
+    const failingForMs = lastSuccessMs !== null
+      ? Math.max(0, Date.now() - lastSuccessMs)
+      : u.consecutiveFailures * PROBE_INTERVAL_MS;
+    if (failingForMs < DEAD_AFTER_MS) return;
+    if (this.openIncidents.has(u.host)) return;
+
+    // Mimic the shape HealthChecker.createIncident expects.
+    try {
+      this.healthChecker.createIncident(u.host, 'caddy-upstream-dead',
+        `Caddy upstream ${u.host} (site ${u.site}) unreachable for ${Math.round(failingForMs / 60000)}m: ${u.lastError || 'no response'}`,
+        {
+          serviceId: u.host,
+          timestamp: u.lastFailureAt,
+          status: 'down',
+          error: u.lastError,
+          details: { site: u.site, siteFile: u.siteFile }
+        }
+      );
+      this.openIncidents.add(u.host);
+      this.emit('upstream-dead', u);
+      this.log.warn?.('caddy-upstream-watcher', `upstream dead: ${u.host} (${u.site})`);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `incident create failed: ${e.message}`);
+    }
+  }
+
+  _maybeResolve(u) {
+    if (!this.healthChecker) return;
+    if (!this.openIncidents.has(u.host)) return;
+    const downSince = u.lastFailureAt ? new Date(u.lastFailureAt).getTime() : 0;
+    const recoveredForMs = downSince ? Date.now() - downSince : 0;
+    if (recoveredForMs < RESOLVED_AFTER_MS) return;
+    try {
+      this.healthChecker.resolveIncident(u.host, 'caddy-upstream-dead', {
+        serviceId: u.host,
+        timestamp: u.lastSuccessAt || new Date().toISOString(),
+        status: 'up'
+      });
+      this.openIncidents.delete(u.host);
+      this.emit('upstream-recovered', u);
+      this.log.info?.('caddy-upstream-watcher', `upstream recovered: ${u.host}`);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `incident resolve failed: ${e.message}`);
+    }
+  }
+
+  _doProbe(host, port) {
+    return new Promise((resolve) => {
+      const isHttps = port === '443';
+      const lib = isHttps ? https : http;
+      const opts = {
+        hostname: host,
+        port: port || (isHttps ? 443 : 80),
+        method: 'HEAD',
+        path: '/',
+        timeout: PROBE_TIMEOUT_MS,
+        headers: { 'X-DashCaddy-HealthCheck': '1', 'User-Agent': 'DashCaddy-CaddyUpstreamWatcher/1' },
+        rejectUnauthorized: false
+      };
+      const req = lib.request(opts, (res) => {
+        res.resume();
+        const healthy = HEALTHY_CODES.has(res.statusCode);
+        resolve({ healthy, statusCode: res.statusCode });
+      });
+      req.on('timeout', () => {
+        req.destroy(new Error('probe timeout'));
+      });
+      req.on('error', (err) => {
+        resolve({ healthy: false, error: err.message });
+      });
+      req.end();
+    });
+  }
+
+  /** Public snapshot for the API/UI. */
+  snapshot() {
+    const list = [];
+    for (const u of this.upstreams.values()) {
+      const muted = this.muted.has(u.host);
+      // Same anchor as _maybeOpenIncident: time since the last successful
+      // probe. If we've never seen a success, fall back to consecutive
+      // failures × probe interval as a worst-case lower bound.
+      const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
+      let failingFor = 0;
+      if (!muted) {
+        if (lastSuccessMs !== null) {
+          failingFor = Math.max(0, Date.now() - lastSuccessMs);
+        } else if (u.status === 'down') {
+          failingFor = u.consecutiveFailures * PROBE_INTERVAL_MS;
+        }
+      }
+      list.push({
+        host: u.host,
+        site: u.site,
+        siteFile: u.siteFile,
+        status: muted ? 'muted' : u.status,
+        consecutiveFailures: u.consecutiveFailures,
+        lastCheckedAt: u.lastCheckedAt,
+        lastSuccessAt: u.lastSuccessAt,
+        lastFailureAt: u.lastFailureAt,
+        lastError: u.lastError,
+        failingForMs: failingFor,
+        muted,
+        dead: !muted && failingFor >= DEAD_AFTER_MS
+      });
+    }
+    // Sort: dead first, then down, then up, then unknown. Within each, by host.
+    list.sort((a, b) => {
+      const order = { dead: 0, down: 1, muted: 2, up: 3, unknown: 4 };
+      const oa = order[a.dead ? 'dead' : a.status] ?? 9;
+      const ob = order[b.dead ? 'dead' : b.status] ?? 9;
+      if (oa !== ob) return oa - ob;
+      return a.host.localeCompare(b.host);
+    });
+    return {
+      upstreams: list,
+      config: {
+        probeIntervalMs: PROBE_INTERVAL_MS,
+        deadAfterMs: DEAD_AFTER_MS,
+        resolvedAfterMs: RESOLVED_AFTER_MS,
+        sitesDir: SITES_DIR
+      }
+    };
+  }
+
+  setMuted(host, muted) {
+    if (muted) {
+      this.muted.add(host);
+    } else {
+      this.muted.delete(host);
+      // Reset failure state on unmute so we don't immediately re-incident a
+      // upstream that just came off mute.
+      const u = this.upstreams.get(host);
+      if (u) {
+        u.consecutiveFailures = 0;
+        u.lastError = null;
+        u.lastFailureAt = null;
+        u.status = 'unknown';
+      }
+    }
+    this._saveState();
+    return { host, muted: !!muted };
+  }
+
+  isMuted(host) { return this.muted.has(host); }
+
+  _loadState() {
+    try {
+      if (!fs.existsSync(STATE_FILE)) return;
+      const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
+      if (Array.isArray(data.muted)) this.muted = new Set(data.muted);
+      // Don't reload upstreams from disk — sites dir is the source of truth.
+      // But preserve last-check state for hosts that still exist.
+      if (data.upstreams && typeof data.upstreams === 'object') {
+        this._restoreUpstreamStates(data.upstreams);
+      }
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `state load failed: ${e.message}`);
+    }
+  }
+
+  _restoreUpstreamStates(persisted) {
+    for (const [host, st] of Object.entries(persisted)) {
+      if (this.upstreams.has(host)) continue;
+      this.upstreams.set(host, {
+        host,
+        ip: st.ip || host.split(':')[0],
+        port: st.port || null,
+        site: st.site || '',
+        siteFile: st.siteFile || '',
+        consecutiveFailures: st.consecutiveFailures || 0,
+        lastFailureAt: st.lastFailureAt || null,
+        lastSuccessAt: st.lastSuccessAt || null,
+        lastError: st.lastError || null,
+        lastCheckedAt: st.lastCheckedAt || null,
+        status: 'unknown'
+      });
+    }
+  }
+
+  _saveState() {
+    try {
+      const dir = path.dirname(STATE_FILE);
+      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+      const upstreams = {};
+      for (const [k, v] of this.upstreams.entries()) {
+        upstreams[k] = {
+          ip: v.ip,
+          port: v.port,
+          site: v.site,
+          siteFile: v.siteFile,
+          consecutiveFailures: v.consecutiveFailures,
+          lastFailureAt: v.lastFailureAt,
+          lastSuccessAt: v.lastSuccessAt,
+          lastError: v.lastError,
+          lastCheckedAt: v.lastCheckedAt
+        };
+      }
+      const tmp = STATE_FILE + '.tmp';
+      fs.writeFileSync(tmp, JSON.stringify({ muted: Array.from(this.muted), upstreams }, null, 2));
+      fs.renameSync(tmp, STATE_FILE);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `state save failed: ${e.message}`);
+    }
+  }
+}
+
+// Singleton — matches the pattern of health-checker.js so it integrates
+// without a separate instantiation site.
+module.exports = new CaddyUpstreamWatcher();
+module.exports.CaddyUpstreamWatcher = CaddyUpstreamWatcher;