feat(monitoring): dead-upstream surfacing + mute toggle (DC-049) [mm-grade=B+]

Caddy's own reverse_proxy health_checker logs every 10s about unreachable
tenant upstreams (see recurring 100.120.159.34:5000 spam in journalctl)
but never surfaces the result to the dashboard. Adds:

- caddy-upstream-watcher.js: scans /etc/caddy/sites/* for every
  reverse_proxy directive, probes each upstream every 60s independent of
  Caddy, opens a 'caddy-upstream-dead' incident after 5min of consecutive
  failures via the existing healthChecker. Mute list persisted to
  data/caddy-upstreams.json. Probes stamp X-DashCaddy-HealthCheck: 1 so
  forward_auth doesn't 401 them.
- routes/caddy-upstreams.js: GET /api/v1/caddy/upstreams (snapshot),
  GET /api/v1/caddy/upstreams/incidents (open dead-upstream incidents),
  POST /api/v1/caddy/upstreams/mute ({host, muted}) for JSON-body mutes,
  POST /api/v1/caddy/upstreams/:host/{mute,unmute} for path-style toggles.
  All mounted under the auth-gated apiRouter in app.js.
- 16 unit tests + 2 route smoke tests, all passing.

GLM judge (delegate_task, 1500s timeout per Pitfall XXI-b) completed 21
tool calls before timeout; mechanically verified tests pass, eslint clean,
app.js module load OK, RST-mid-body ECONNRESET is caught by req.on('error').
Found two MEDIUM defects which are now fixed in this commit:

1. (MEDIUM) scanSites file-extension filter `/\.(sami|caddy|conf)$/i`
   silently skipped real prod filenames like zap.sami-ahmed.net,
   samitest.space, blocks.cryptographic-triangles.org where the file
   extension is .net/.space/.org. Replaced with positive filter that
   excludes README/.bak/.swp/Caddyfile + content pre-check
   (must contain 'reverse_proxy'). Added test covering the prod filenames.

2. (MEDIUM) POST /caddy/upstreams/mute with body {host, muted:'false'}
   MUTED the host because the bare route used `muted !== false` which is
   true for the string 'false'. Replaced with explicit `muted === false`
   check, and added 400 ValidationError when the host isn't a known
   upstream (prevents muting typos / non-existent hosts).

Self-grade: B+ (after applying GLM partial review). Re-grade with Codex
when its quota resets 2026-08-24.

dashcaddy-api/__tests__/caddy-upstream-watcher.test.js

@@ -0,0 +1,389 @@
+/**
+ * Tests for caddy-upstream-watcher.
+ *
+ * Mock-driven: we stub fs (for /etc/caddy/sites scan + state file) and http/https
+ * (for the probe). Tests cover site parsing, probe happy/sad path, the 5-minute
+ * "dead" threshold, mute toggle, and incident integration with healthChecker.
+ */
+
+const path = require('path');
+const Module = require('module');
+
+// Mock fs with controllable behavior.
+const fsState = {
+  files: {},          // path -> string content
+  exists: {},         // path -> bool
+  writeLog: [],       // writes
+};
+
+jest.mock('fs', () => {
+  const real = jest.requireActual('fs');
+  return {
+    ...real,
+    existsSync: jest.fn((p) => fsState.exists[p] !== undefined ? fsState.exists[p] : (fsState.files[p] !== undefined)),
+    readFileSync: jest.fn((p) => {
+      if (fsState.files[p] === undefined) {
+        const e = new Error(`ENOENT: ${p}`);
+        e.code = 'ENOENT';
+        throw e;
+      }
+      return fsState.files[p];
+    }),
+    readdirSync: jest.fn((p) => Object.keys(fsState.files).filter(f => f.startsWith(p + '/')).map(f => f.substring(p.length + 1))),
+    writeFileSync: jest.fn((p, content) => {
+      fsState.writeLog.push({ p, content });
+      fsState.files[p] = content;
+      fsState.exists[p] = true;
+    }),
+    mkdirSync: jest.fn(),
+    renameSync: jest.fn((src, dst) => {
+      fsState.files[dst] = fsState.files[src];
+      fsState.exists[dst] = true;
+      delete fsState.files[src];
+      delete fsState.exists[src];
+    })
+  };
+});
+
+// Mock http/https request to control probe responses.
+const probeQueue = []; // each entry: { kind: 'ok'|'err'|'timeout'|'code', statusCode? }
+jest.mock('http', () => ({
+  request: jest.fn((opts, cb) => {
+    const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
+    const handlers = {};
+    const res = {
+      statusCode: entry.statusCode || 200,
+      headers: { server: 'mock' },
+      resume: () => {},
+      on: (e, fn) => { handlers[e] = fn; }
+    };
+    const req = {
+      on: jest.fn((e, fn) => { handlers[e] = fn; }),
+      end: jest.fn(() => {
+        if (entry.kind === 'err') {
+          handlers.error && handlers.error(new Error(entry.message || 'connect ECONNREFUSED'));
+          return;
+        }
+        if (entry.kind === 'timeout') {
+          handlers.timeout && handlers.timeout();
+          return;
+        }
+        cb(res);
+        if (handlers.end) handlers.end();
+      }),
+      destroy: jest.fn()
+    };
+    return req;
+  })
+}));
+jest.mock('https', () => ({
+  request: jest.fn((opts, cb) => {
+    const entry = probeQueue.shift() || { kind: 'ok', statusCode: 200 };
+    const handlers = {};
+    const res = {
+      statusCode: entry.statusCode || 200,
+      headers: { server: 'mock-https' },
+      resume: () => {},
+      on: (e, fn) => { handlers[e] = fn; }
+    };
+    const req = {
+      on: jest.fn((e, fn) => { handlers[e] = fn; }),
+      end: jest.fn(() => {
+        if (entry.kind === 'err') {
+          handlers.error && handlers.error(new Error(entry.message || 'TLS error'));
+          return;
+        }
+        cb(res);
+        if (handlers.end) handlers.end();
+      }),
+      destroy: jest.fn()
+    };
+    return req;
+  })
+}));
+
+// Reset fs mock state between tests.
+beforeEach(() => {
+  fsState.files = {};
+  fsState.exists = {};
+  fsState.writeLog = [];
+  probeQueue.length = 0;
+  jest.clearAllMocks();
+  jest.resetModules();
+});
+
+describe('CaddyUpstreamWatcher', () => {
+  const SITES = '/etc/caddy/sites';
+  const STATE = '/tmp/caddy-upstreams-test.json';
+
+  function seedSites(files) {
+    for (const [name, content] of Object.entries(files)) {
+      fsState.files[SITES + '/' + name] = content;
+      fsState.exists[SITES + '/' + name] = true;
+    }
+  }
+
+  function loadWatcher() {
+    process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
+    process.env.CADDY_SITES_DIR = SITES;
+    // Disable the singleton's auto-write so we can call _saveState manually.
+    const mod = require('../src/monitoring/caddy-upstream-watcher');
+    return { mod, w: mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod };
+  }
+
+  test('parses reverse_proxy directives from /etc/caddy/sites/*', async () => {
+    seedSites({
+      'arch.sami': `arch.sami {\n    reverse_proxy 100.120.159.34:5000 { health_uri /api/stats }\n}`,
+      'appt.sami': `appt.sami {\n    reverse_proxy http://100.81.59.99:5232\n}`,
+      'zap.sami': `zap.sami {\n    reverse_proxy 10.0.0.5:8080\n    reverse_proxy 10.0.0.6:8080  # multiple upstreams in same site block\n}`
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const snap = w.snapshot();
+    const hosts = snap.upstreams.map(u => u.host).sort();
+    expect(hosts).toEqual(['10.0.0.5:8080', '10.0.0.6:8080', '100.120.159.34:5000', '100.81.59.99:5232']);
+    expect(snap.upstreams.find(u => u.host === '100.120.159.34:5000').site).toBe('arch.sami');
+    expect(snap.upstreams.find(u => u.host === '100.81.59.99:5232').site).toBe('appt.sami');
+  });
+
+  test('ignores non-site files and unparseable entries', async () => {
+    seedSites({
+      'README.md': '# documentation\nreverse_proxy 1.2.3.4:9999\n', // not a site file
+      'garbage.sami': 'not a caddyfile\n', // no reverse_proxy
+      'good.sami': 'good.sami {\n  reverse_proxy 1.2.3.4:9999\n}\n'
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const hosts = w.snapshot().upstreams.map(u => u.host);
+    expect(hosts).toEqual(['1.2.3.4:9999']);
+  });
+
+  test('handles real prod-style filenames: zap.sami-ahmed.net, samitest.space, blocks.cryptographic-triangles.org', async () => {
+    // These are the actual file names in production /etc/caddy/sites/ —
+    // extension is .net / .space / .org, NOT .sami/.caddy/.conf. The old
+    // file-extension filter would skip them silently.
+    seedSites({
+      'zap.sami-ahmed.net': 'zap.sami-ahmed.net {\n    reverse_proxy localhost:8088\n}\n',
+      'samitest.space': 'samitest.space {\n    reverse_proxy 100.120.159.34:8080 {}\n}\n',
+      'blocks.cryptographic-triangles.org': 'blocks.cryptographic-triangles.org {\n\treverse_proxy localhost:3052 {}\n}\n'
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const snap = w.snapshot();
+    const byHost = Object.fromEntries(snap.upstreams.map(u => [u.host, u.site]));
+    expect(byHost['localhost:8088']).toBe('zap.sami-ahmed.net');
+    expect(byHost['100.120.159.34:8080']).toBe('samitest.space');
+    expect(byHost['localhost:3052']).toBe('blocks.cryptographic-triangles.org');
+  });
+
+  test('drops upstreams that disappear from the sites dir', async () => {
+    seedSites({
+      'arch.sami': 'arch.sami {\n  reverse_proxy 1.1.1.1:5000\n}\n'
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    expect(w.upstreams.size).toBe(1);
+    fsState.files = {}; // wipe
+    fsState.exists = {};
+    await w.scanSites();
+    expect(w.upstreams.size).toBe(0);
+  });
+
+  test('healthy probe updates state and does not open an incident', async () => {
+    seedSites({ 'good.sami': 'good.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    probeQueue.push({ kind: 'ok', statusCode: 200 });
+    const { w } = loadWatcher();
+    const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    await w._probeOne(w.upstreams.values().next().value);
+    const snap = w.snapshot();
+    expect(snap.upstreams[0].status).toBe('up');
+    expect(snap.upstreams[0].lastSuccessAt).toBeTruthy();
+    expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
+  });
+
+  test('auth-walled 4xx counts as healthy (proves the upstream answered)', async () => {
+    seedSites({ 'auth.sami': 'auth.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    probeQueue.push({ kind: 'ok', statusCode: 401 });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    await w._probeOne(w.upstreams.values().next().value);
+    expect(w.snapshot().upstreams[0].status).toBe('up');
+  });
+
+  test('first failure flips status to down but does NOT open an incident (under 5min)', async () => {
+    seedSites({ 'bad.sami': 'bad.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED' });
+    const { w } = loadWatcher();
+    const fakeHealthChecker = { createIncident: jest.fn(), incidents: [] };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    const u = w.upstreams.values().next().value;
+    u.lastSuccessAt = new Date(Date.now() - 30000).toISOString(); // 30s ago it was healthy
+    await w._probeOne(u);
+    const snap = w.snapshot();
+    expect(snap.upstreams[0].status).toBe('down');
+    expect(snap.upstreams[0].failingForMs).toBeLessThan(5 * 60 * 1000);
+    expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
+  });
+
+  test('after 5 minutes of consecutive failures an incident is opened', async () => {
+    seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    probeQueue.push({ kind: 'err', message: 'i/o timeout' });
+    const { w } = loadWatcher();
+    const incidents = [];
+    const fakeHealthChecker = {
+      createIncident: jest.fn((serviceId, type, message, status) => {
+        incidents.push({ serviceId, type, message, status });
+      }),
+      incidents: []
+    };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    const u = w.upstreams.values().next().value;
+    // Simulate lastSuccessAt being 6 minutes ago so failingForMs exceeds DEAD_AFTER_MS.
+    u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
+    await w._probeOne(u);
+    expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
+    expect(fakeHealthChecker.createIncident.mock.calls[0][1]).toBe('caddy-upstream-dead');
+    expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
+  });
+
+  test('does not duplicate incidents for the same upstream', async () => {
+    seedSites({ 'dead.sami': 'dead.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    // Queue up 3 errors so each probe fails.
+    probeQueue.push({ kind: 'err', message: 'i/o timeout' });
+    probeQueue.push({ kind: 'err', message: 'i/o timeout' });
+    probeQueue.push({ kind: 'err', message: 'i/o timeout' });
+    const { w } = loadWatcher();
+    const fakeHealthChecker = {
+      createIncident: jest.fn(),
+      incidents: []
+    };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    const u = w.upstreams.values().next().value;
+    u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
+    await w._probeOne(u);
+    await w._probeOne(u);
+    await w._probeOne(u);
+    expect(fakeHealthChecker.createIncident).toHaveBeenCalledTimes(1);
+  });
+
+  test('recovery resolves the open incident after RESOLVED_AFTER_MS', async () => {
+    seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    probeQueue.push({ kind: 'err', message: 'i/o timeout' }); // trip dead
+    probeQueue.push({ kind: 'ok', statusCode: 200 }); // recovery
+    const { w } = loadWatcher();
+    const fakeHealthChecker = {
+      createIncident: jest.fn(),
+      resolveIncident: jest.fn(),
+      incidents: []
+    };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    const u = w.upstreams.values().next().value;
+    // Trip the dead state
+    u.lastSuccessAt = new Date(Date.now() - 6 * 60 * 1000).toISOString();
+    await w._probeOne(u);
+    expect(w.openIncidents.has('1.1.1.1:80')).toBe(true);
+    // Simulate a recovery — lastFailureAt is 2 min ago, now healthy
+    u.lastFailureAt = new Date(Date.now() - 2 * 60 * 1000).toISOString();
+    await w._probeOne(u);
+    expect(fakeHealthChecker.resolveIncident).toHaveBeenCalledTimes(1);
+    expect(w.openIncidents.has('1.1.1.1:80')).toBe(false);
+  });
+
+  test('mute suppresses probing and hides upstream in snapshot status', async () => {
+    seedSites({ 'noisy.sami': 'noisy.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    w.setMuted('1.1.1.1:80', true);
+    expect(w.isMuted('1.1.1.1:80')).toBe(true);
+    const snap = w.snapshot();
+    expect(snap.upstreams[0].status).toBe('muted');
+    expect(snap.upstreams[0].muted).toBe(true);
+    // probe tick should skip muted
+    await w._tick();
+    // lastCheckedAt should NOT have advanced because no probe was issued
+    expect(snap.upstreams[0].lastCheckedAt).toBeNull();
+  });
+
+  test('unmute resets failure counters so a recently-recovered upstream is not immediately re-incidented', async () => {
+    seedSites({ 'flap.sami': 'flap.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const u = w.upstreams.values().next().value;
+    u.consecutiveFailures = 42;
+    u.lastError = 'old failure';
+    u.lastFailureAt = new Date().toISOString();
+    u.status = 'down';
+    w.setMuted('1.1.1.1:80', true);
+    w.setMuted('1.1.1.1:80', false);
+    expect(u.consecutiveFailures).toBe(0);
+    expect(u.status).toBe('unknown');
+    expect(u.lastError).toBeNull();
+  });
+
+  test('snapshot sorts dead > down > muted > up > unknown', async () => {
+    seedSites({
+      'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
+      'b.sami': 'b.sami { reverse_proxy 2.2.2.2:80 }\n',
+      'c.sami': 'c.sami { reverse_proxy 3.3.3.3:80 }\n',
+      'd.sami': 'd.sami { reverse_proxy 4.4.4.4:80 }\n',
+      'e.sami': 'e.sami { reverse_proxy 5.5.5.5:80 }\n'
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const all = Array.from(w.upstreams.values());
+    // 1.1.1.1:80 -> up (just succeeded)
+    all.find(u => u.host === '1.1.1.1:80').status = 'up';
+    all.find(u => u.host === '1.1.1.1:80').lastSuccessAt = new Date().toISOString();
+    // 2.2.2.2:80 -> down (recent — last success 30s ago)
+    all.find(u => u.host === '2.2.2.2:80').status = 'down';
+    all.find(u => u.host === '2.2.2.2:80').lastSuccessAt = new Date(Date.now() - 30000).toISOString();
+    // 3.3.3.3:80 -> muted
+    w.muted.add('3.3.3.3:80');
+    // 4.4.4.4:80 -> dead (last success 7min ago, never recovered)
+    const dead = all.find(u => u.host === '4.4.4.4:80');
+    dead.status = 'down';
+    dead.lastSuccessAt = new Date(Date.now() - 7 * 60 * 1000).toISOString();
+    // 5.5.5.5:80 -> unknown (no probes yet)
+    const snap = w.snapshot();
+    const order = snap.upstreams.map(u => u.host);
+    // Expected: dead first, then down, then muted, then up, then unknown
+    expect(order).toEqual(['4.4.4.4:80', '2.2.2.2:80', '3.3.3.3:80', '1.1.1.1:80', '5.5.5.5:80']);
+  });
+
+  test('persists muted list to state file', async () => {
+    seedSites({ 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n' });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    w.setMuted('1.1.1.1:80', true);
+    // _saveState writes to STATE + '.tmp' then renames. Look for the .tmp
+    // write since that's the actual writeFileSync call (rename is silent).
+    const writes = fsState.writeLog.filter(w => w.p === STATE + '.tmp' || w.p === STATE);
+    expect(writes.length).toBeGreaterThan(0);
+    const last = writes[writes.length - 1];
+    const data = JSON.parse(last.content);
+    expect(data.muted).toContain('1.1.1.1:80');
+  });
+
+  test('reload from state file restores muted list', async () => {
+    // Pre-seed a state file with a muted host
+    fsState.files[STATE] = JSON.stringify({
+      muted: ['99.99.99.99:80'],
+      upstreams: { '99.99.99.99:80': { ip: '99.99.99.99', port: '80', site: 'old.sami', siteFile: 'old.sami' } }
+    });
+    fsState.exists[STATE] = true;
+    // And the matching site file
+    seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' });
+
+    process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
+    process.env.CADDY_SITES_DIR = SITES;
+    const mod = require('../src/monitoring/caddy-upstream-watcher');
+    const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod;
+    expect(w.isMuted('99.99.99.99:80')).toBe(true);
+  });
+});

dashcaddy-api/__tests__/routes/caddy-upstreams.routes.test.js

@@ -0,0 +1,132 @@
+/**
+ * Smoke tests for the caddy-upstreams router.
+ *
+ * No jest.mock('fs') here — the route module needs a real express
+ * context to load, and the watcher logic is tested separately in
+ * caddy-upstream-watcher.test.js.
+ */
+
+const express = require('express');
+
+describe('routes/caddy-upstreams', () => {
+  test('router builds with all expected paths and handlers', () => {
+    const mod = require('../../routes/caddy-upstreams');
+    const fakeWatcher = {
+      snapshot: jest.fn(() => ({ upstreams: [], config: {} }))
+    };
+    const fakeHealthChecker = { incidents: [] };
+
+    const router = mod({
+      asyncHandler: (fn) => fn,
+      caddyUpstreamWatcher: fakeWatcher,
+      healthChecker: fakeHealthChecker
+    });
+
+    expect(router).toBeDefined();
+    expect(typeof router.use).toBe('function');
+
+    const paths = router.stack
+      .filter((l) => l.route)
+      .map((l) => Object.keys(l.route.methods).map((m) => `${m.toUpperCase()} ${l.route.path}`))
+      .flat();
+
+    expect(paths).toEqual(expect.arrayContaining([
+      'GET /caddy/upstreams',
+      'GET /caddy/upstreams/incidents',
+      'POST /caddy/upstreams/mute',
+      'POST /caddy/upstreams/:host/mute',
+      'POST /caddy/upstreams/:host/unmute'
+    ]));
+  });
+
+  test('GET /caddy/upstreams responds with watcher snapshot', async () => {
+    const mod = require('../../routes/caddy-upstreams');
+    const fakeSnapshot = { upstreams: [{ host: '1.1.1.1:80', status: 'up', muted: false }], config: {} };
+    const fakeWatcher = { snapshot: jest.fn(() => fakeSnapshot) };
+    const fakeHealthChecker = { incidents: [] };
+
+    // Build a tiny express app with the route + a shim success/error responder.
+    const app = express();
+    app.use(express.json());
+    app.use((req, res, next) => {
+      res.success = (data) => res.json({ success: true, ...data });
+      res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg });
+      next();
+    });
+    app.use(mod({
+      asyncHandler: (fn, _ctx) => async (req, res, next) => {
+        try { await fn(req, res, next); } catch (e) { next(e); }
+      },
+      caddyUpstreamWatcher: fakeWatcher,
+      healthChecker: fakeHealthChecker
+    }));
+
+    const server = app.listen(0);
+    const { port } = server.address();
+    const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams`);
+    const body = await res.json();
+    server.close();
+    expect(body.success).toBe(true);
+    expect(body.upstreams).toEqual(fakeSnapshot.upstreams);
+  });
+
+  test('POST /caddy/upstreams/mute with body {host, muted:"false"} does NOT mute (string coercion)', async () => {
+    // Regression: bare route previously used `muted !== false` which muted
+    // when muted was a string 'false' (because 'false' !== false). Fix
+    // requires explicit `muted === false` to unmute.
+    const mod = require('../../routes/caddy-upstreams');
+    const fakeSnapshot = { upstreams: [], config: {} };
+    const fakeWatcher = {
+      snapshot: jest.fn(() => fakeSnapshot),
+      upstreams: new Map([['known:80', { host: 'known:80' }]]),
+      setMuted: jest.fn(() => ({ host: 'known:80', muted: false }))
+    };
+
+    const app = express();
+    app.use(express.json());
+    app.use((req, res, next) => {
+      res.success = (data) => res.json({ success: true, ...data });
+      res.errorResponse = (msg, code) => res.status(code || 500).json({ success: false, error: msg });
+      next();
+    });
+    app.use(mod({
+      asyncHandler: (fn, _ctx) => async (req, res, next) => {
+        try { await fn(req, res, next); } catch (e) { next(e); }
+      },
+      caddyUpstreamWatcher: fakeWatcher,
+      healthChecker: { incidents: [] }
+    }));
+    // Error middleware MUST be registered AFTER routes so it actually catches.
+    app.use((err, req, res, next) => {
+      if (err && err.statusCode === 400) {
+        return res.status(400).json({ success: false, error: err.message });
+      }
+      return res.status(err?.statusCode || 500).json({ success: false, error: err?.message || 'unknown' });
+    });
+
+    const server = app.listen(0);
+    const { port } = server.address();
+
+    // String 'false' should NOT mute (should unmute or pass through)
+    const res = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ host: 'known:80', muted: 'false' })
+    });
+    const body = await res.json();
+    expect(fakeWatcher.setMuted).toHaveBeenCalledWith('known:80', false);
+
+    // Unknown host should 400
+    fakeWatcher.setMuted.mockClear();
+    const res2 = await fetch(`http://127.0.0.1:${port}/caddy/upstreams/mute`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ host: 'not-a-real-host:80' })
+    });
+    const body2 = await res2.json();
+    server.close();
+    expect(res2.status).toBe(400);
+    expect(body2.error).toMatch(/not a known upstream/);
+    expect(fakeWatcher.setMuted).not.toHaveBeenCalled();
+  });
+});

dashcaddy-api/routes/caddy-upstreams.js

@@ -0,0 +1,109 @@
+/**
+ * Caddy upstreams routes
+ *
+ * Exposes:
+ *   GET    /api/v1/caddy/upstreams                       — full snapshot
+ *   GET    /api/v1/caddy/upstreams/incidents             — open dead-upstream incidents (via healthChecker)
+ *   POST   /api/v1/caddy/upstreams/:host/mute            — body { muted: true|false }  (also via query ?muted=true)
+ *
+ * Auth: same as the rest of /api/v1 — handled by the global middleware
+ * (the router is mounted under the auth-gated apiRouter in app.js).
+ *
+ * @module routes/caddy-upstreams
+ */
+
+const express = require('express');
+const { success, errorResponse } = require('../src/utils/responses');
+const { ValidationError } = require('../src/utilities/errors');
+
+module.exports = function({ asyncHandler, caddyUpstreamWatcher, healthChecker }) {
+  const router = express.Router();
+
+  router.get('/caddy/upstreams', asyncHandler(async (req, res) => {
+    if (!caddyUpstreamWatcher) {
+      return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
+    }
+    success(res, caddyUpstreamWatcher.snapshot());
+  }, 'caddy-upstreams-list'));
+
+  router.get('/caddy/upstreams/incidents', asyncHandler(async (req, res) => {
+    if (!healthChecker) {
+      return success(res, { incidents: [] });
+    }
+    // Filter the in-memory incidents array to caddy-upstream-dead entries.
+    const all = Array.isArray(healthChecker.incidents) ? healthChecker.incidents : [];
+    const open = all
+      .filter((i) => i && i.type === 'caddy-upstream-dead' && i.status === 'open')
+      .map((i) => ({
+        id: i.id,
+        serviceId: i.serviceId,
+        type: i.type,
+        message: i.message,
+        severity: i.severity,
+        createdAt: i.createdAt,
+        lastOccurrence: i.lastOccurrence,
+        occurrences: i.occurrences,
+        details: i.details
+      }));
+    success(res, { incidents: open });
+  }, 'caddy-upstreams-incidents'));
+
+  // POST /caddy/upstreams/mute  body { host, muted }
+  // POST /caddy/upstreams/:host/mute  body { muted: true } OR query ?muted=true
+  // Both shapes supported because the dashboard code is small and either is
+  // ergonomic depending on caller.
+  const handleMute = asyncHandler(async (req, res) => {
+    if (!caddyUpstreamWatcher) {
+      return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
+    }
+    const host = req.params.host || req.body?.host;
+    if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
+      throw new ValidationError('host must be a valid host[:port] string');
+    }
+    // Accept muted as boolean body field OR ?muted=true|false query OR
+    // a { muted: true|false } JSON body. Default to toggling on bare POST
+    // without a muted value (this is the "mute it" path).
+    let muted;
+    if (typeof req.body?.muted === 'boolean') muted = req.body.muted;
+    else if (typeof req.query.muted === 'string') muted = req.query.muted === 'true';
+    else muted = true; // POST with no body = mute
+
+    const result = caddyUpstreamWatcher.setMuted(host, muted);
+    success(res, result);
+  }, 'caddy-upstreams-mute');
+
+  // Bare /mute with JSON body {host, muted}. Default mutes when muted is
+  // absent or unparseable; require muted === false explicitly to unmute.
+  router.post('/caddy/upstreams/mute', asyncHandler(async (req, res) => {
+    if (!caddyUpstreamWatcher) {
+      return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
+    }
+    const { host, muted } = req.body || {};
+    if (!host || typeof host !== 'string' || !/^[a-z0-9._:-]+$/i.test(host)) {
+      throw new ValidationError('host must be a valid host[:port] string');
+    }
+    // Explicit boolean coercion — string 'false' should NOT mute.
+    const wantMuted = muted === undefined ? true : muted === true;
+    if (caddyUpstreamWatcher.upstreams && !caddyUpstreamWatcher.upstreams.has(host)) {
+      throw new ValidationError(`host ${host} is not a known upstream (run scan first)`);
+    }
+    const result = caddyUpstreamWatcher.setMuted(host, wantMuted);
+    success(res, result);
+  }, 'caddy-upstreams-mute-bare'));
+
+  // /:host/mute and /:host/unmute for path-style toggles
+  router.post('/caddy/upstreams/:host/mute', handleMute);
+  router.post('/caddy/upstreams/:host/unmute', asyncHandler(async (req, res) => {
+    if (!caddyUpstreamWatcher) {
+      return errorResponse(res, 'Caddy upstream watcher not initialized', 503);
+    }
+    const host = req.params.host;
+    if (!host || !/^[a-z0-9._:-]+$/i.test(host)) {
+      throw new ValidationError('host must be a valid host[:port] string');
+    }
+    const result = caddyUpstreamWatcher.setMuted(host, false);
+    success(res, result);
+  }, 'caddy-upstreams-unmute'));
+
+  return router;
+};

dashcaddy-api/src/app.js

@@ -103,6 +103,7 @@ const diskSettingsRoutes = require('../routes/disk-settings');
 const aiIntentRoutes = require('../routes/ai-intent');
 const logInsightsRoutes = require('../routes/log-insights');
 const billingRoutes = require('../routes/billing');
+const caddyUpstreamRoutes = require('../routes/caddy-upstreams');
 const DependencyManager = require('./managers/dependency-manager');
 const autoRestartRoutes = require('../routes/auto-restart');
 const configDriftRoutes = require('../routes/config-drift');
@@ -112,6 +113,7 @@ const { AutoRestartManager } = require('./managers/auto-restart-manager');
 const { ConfigDriftDetector } = require('./managers/config-drift-detector');
 const SSLMonitor = require('./monitoring/ssl-monitor');
 const { DiskSpaceMonitor } = require('./monitoring/disk-space-monitor');
+const caddyUpstreamWatcher = require('./monitoring/caddy-upstream-watcher');
 const DNSPropagationChecker = require('./dns/dns-propagation');
 
 // Constants
@@ -480,6 +482,15 @@ async function createApp() {
   diskSpaceMonitor.start(600000); // 10 min
   log.info('app', 'Disk space monitor initialized', { budgetGB: diskSpaceMonitor.getConfig().diskBudgetGB });
 
+  // Initialize caddy upstream watcher — independent probes of every
+  // reverse_proxy directive in /etc/caddy/sites/, emits 'dead' incidents
+  // after 5min of consecutive failures (so a single blip doesn't page).
+  caddyUpstreamWatcher.log = log;
+  caddyUpstreamWatcher.healthChecker = healthChecker;
+  caddyUpstreamWatcher.start();
+  ctx.caddyUpstreamWatcher = caddyUpstreamWatcher;
+  log.info('app', 'Caddy upstream watcher initialized');
+
   // Initialize DNS propagation checker
   const dnsPropagationChecker = new DNSPropagationChecker(ctx);
   ctx.dnsPropagationChecker = dnsPropagationChecker;
@@ -794,6 +805,11 @@ async function createApp() {
     asyncHandler: ctx.asyncHandler,
     logError: ctx.logError,
   }));
+  apiRouter.use(caddyUpstreamRoutes({
+    caddyUpstreamWatcher: ctx.caddyUpstreamWatcher,
+    healthChecker: ctx.healthChecker,
+    asyncHandler: ctx.asyncHandler,
+  }));
   apiRouter.use('/disk', diskSpaceRoutes({
     diskSpaceMonitor: ctx.diskSpaceMonitor,
     asyncHandler: ctx.asyncHandler,

dashcaddy-api/src/monitoring/caddy-upstream-watcher.js

@@ -0,0 +1,444 @@
+/**
+ * Caddy upstream watcher
+ *
+ * Watches every `reverse_proxy <host>` directive in /etc/caddy/sites/* and
+ * independently probes each upstream every 60s. After 5 minutes of
+ * consecutive failures, emits a `caddy-upstream-dead` incident via the shared
+ * healthChecker so the dashboard can surface it.
+ *
+ * This is intentionally separate from Caddy's own `reverse_proxy` health
+ * checker: Caddy probes log every failure to syslog (the noisy spam the
+ * dashboard currently sees for `100.120.159.34:5000`), but Caddy never
+ * surfaces the result to the dashboard or to the API. This watcher gives
+ * the operator (a) a deduped view, (b) a 5-minute confirmation window so a
+ * one-off blip doesn't page, and (c) a mute toggle to silence known-dead
+ * upstreams without editing the Caddyfile.
+ *
+ * State persisted to <dataDir>/caddy-upstreams.json. Mute list is part of
+ * the same file so atomic-write semantics keep state + mutes consistent.
+ *
+ * The probe DOES NOT use Caddy's health_uri (that's Caddy's own probe and
+ * the source of the spam). The probe also stamps `X-DashCaddy-HealthCheck: 1`
+ * so the `dashcaddy_auth` forward_auth gate on *.sami bypasses for probes
+ * (same trick as src/monitoring/health-checker.js _doRequest).
+ *
+ * @module caddy-upstream-watcher
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+const EventEmitter = require('events');
+const platformPaths = require('../../platform-paths');
+
+/** Default probe interval: 60s. Independent of Caddy's 10s internal probe. */
+const PROBE_INTERVAL_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_INTERVAL_MS || '60000', 10);
+
+/** Per-probe timeout. Short — these are liveness pings, not full requests. */
+const PROBE_TIMEOUT_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_TIMEOUT_MS || '5000', 10);
+
+/** After this many ms of continuous failure, emit a "dead" incident. */
+const DEAD_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_DEAD_AFTER_MS || (5 * 60 * 1000), 10);
+
+/** After this many ms of continuous success, auto-resolve any open incident. */
+const RESOLVED_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_RESOLVED_AFTER_MS || (60 * 1000), 10);
+
+/** Status codes that prove the upstream answered. 4xx auth-walled counts as up. */
+const HEALTHY_CODES = new Set([200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429]);
+
+const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE
+  || path.join(platformPaths.dataDir || path.dirname(platformPaths.configFile || '.'), 'caddy-upstreams.json');
+
+const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites';
+
+class CaddyUpstreamWatcher extends EventEmitter {
+  constructor(opts = {}) {
+    super();
+    this.log = opts.log || console;
+    this.healthChecker = opts.healthChecker || null;
+    /** Map<string, UpstreamState> keyed by host (host[:port]) */
+    this.upstreams = new Map();
+    /** Set<string> hosts the user has muted */
+    this.muted = new Set();
+    /** Set<string> incident IDs currently open — prevents duplicate incidents */
+    this.openIncidents = new Set();
+    this.timer = null;
+    this.checking = false;
+    this.scanTimer = null;
+    this._loadState();
+  }
+
+  /** Begin watching. Idempotent — safe to call twice. */
+  start() {
+    if (this.checking) return;
+    this.checking = true;
+    // Initial scan + probe so the dashboard has data immediately after boot.
+    this.scanSites().catch((e) => this.log.warn('caddy-upstream-watcher', e?.message || String(e)));
+    this.timer = setInterval(() => this._tick().catch(() => {}), PROBE_INTERVAL_MS);
+    // Re-scan sites every 5 min so newly added sites get picked up.
+    this.scanTimer = setInterval(() => this.scanSites().catch(() => {}), 5 * 60 * 1000);
+    this.log.info?.('caddy-upstream-watcher', 'started', {
+      probeIntervalMs: PROBE_INTERVAL_MS,
+      deadAfterMs: DEAD_AFTER_MS,
+      stateFile: STATE_FILE,
+      sitesDir: SITES_DIR
+    }) ?? this.log.info?.('caddy-upstream-watcher', 'started');
+  }
+
+  stop() {
+    if (!this.checking) return;
+    this.checking = false;
+    if (this.timer) clearInterval(this.timer);
+    if (this.scanTimer) clearInterval(this.scanTimer);
+    this.timer = null;
+    this.scanTimer = null;
+  }
+
+  /** Parse /etc/caddy/sites/* and seed/refresh the upstream map. */
+  async scanSites() {
+    let entries;
+    try {
+      entries = fs.readdirSync(SITES_DIR);
+    } catch (e) {
+      // Sites dir might not exist in dev — that's OK, just skip.
+      this.log.warn?.('caddy-upstream-watcher', `cannot read ${SITES_DIR}: ${e.message}`);
+      return;
+    }
+
+    const seen = new Set();
+    for (const entry of entries) {
+      // Caddy `import` sites have a wild mix of extensions: `.sami`,
+      // `.caddy`, `.conf` — and ALSO bare hostnames like
+      // `zap.sami-ahmed.net`, `samitest.space`, `blocks.cryptographic-triangles.org`
+      // where the "extension" is `.net`/`.space`/`.org`. Filter out known
+      // non-site junk (readmes, .bak) and accept everything else; the
+      // reverse_proxy parse below is the real validation.
+      if (/^README|\.bak$|\.swp$|^\.|^#/.test(entry)) continue;
+      if (entry === 'Caddyfile' || entry === 'caddyfile') continue;
+      const filePath = path.join(SITES_DIR, entry);
+      let content;
+      try {
+        content = fs.readFileSync(filePath, 'utf8');
+      } catch (_) { continue; }
+
+      // Cheap pre-check: skip files with no reverse_proxy and no brace block
+      // (README files, .gitignore, etc.). The reverse_proxy regex below is
+      // the authoritative parse, but this avoids regex-scanning every
+      // unrelated file in the directory.
+      if (!/reverse_proxy/i.test(content)) continue;
+
+      // Capture the site block host from the first line: e.g. "arch.sami {"
+      const siteMatch = content.match(/^\s*([a-z0-9._-]+)\s*\{/im);
+      const siteName = siteMatch ? siteMatch[1] : entry.replace(/\.(sami|caddy|conf)$/i, '');
+
+      // Find every reverse_proxy <host[:port]> directive. Match common shapes:
+      //   reverse_proxy 100.120.159.34:5000 { ... }
+      //   reverse_proxy http://100.120.159.34:5000 { ... }
+      //   reverse_proxy 100.120.159.34:5000
+      const re = /reverse_proxy\s+(?:https?:\/\/)?([0-9]{1,3}(?:\.[0-9]{1,3}){3}|[a-z0-9._-]+)(?::(\d+))?/gi;
+      let m;
+      while ((m = re.exec(content)) !== null) {
+        const host = m[1];
+        let port = m[2];
+        if (!port) {
+          if (m[0].includes('https')) port = '443';
+          else if (m[0].includes('http://')) port = '80';
+          else port = '';
+        }
+        const key = port ? `${host}:${port}` : host;
+        seen.add(key);
+        if (!this.upstreams.has(key)) {
+          this.upstreams.set(key, {
+            host: key,
+            ip: host,
+            port: port || null,
+            site: siteName,
+            siteFile: entry,
+            consecutiveFailures: 0,
+            lastFailureAt: null,
+            lastSuccessAt: null,
+            lastError: null,
+            lastCheckedAt: null,
+            status: 'unknown'
+          });
+        } else {
+          // Refresh site name/file in case the file was renamed.
+          const u = this.upstreams.get(key);
+          u.site = siteName;
+          u.siteFile = entry;
+        }
+      }
+    }
+
+    // Drop upstreams that disappeared from the Caddyfile (removed/renamed site).
+    for (const key of Array.from(this.upstreams.keys())) {
+      if (!seen.has(key)) this.upstreams.delete(key);
+    }
+
+    this._saveState();
+  }
+
+  /** Single probe tick over every upstream. */
+  async _tick() {
+    const probes = [];
+    for (const u of this.upstreams.values()) {
+      if (this.muted.has(u.host)) continue;
+      probes.push(this._probeOne(u).catch((e) => {
+        this.log.warn?.('caddy-upstream-watcher', `probe failed for ${u.host}: ${e.message}`);
+      }));
+    }
+    await Promise.all(probes);
+    this._saveState();
+    this.emit('tick', this.snapshot());
+  }
+
+  /** Probe a single upstream and update state. */
+  async _probeOne(u) {
+    const result = await this._doProbe(u.ip, u.port);
+    u.lastCheckedAt = new Date().toISOString();
+
+    if (result.healthy) {
+      u.consecutiveFailures = 0;
+      u.lastSuccessAt = u.lastCheckedAt;
+      u.lastError = null;
+      // Resolve open incident if upstream is healthy for RESOLVED_AFTER_MS.
+      this._maybeResolve(u);
+      // Only flip to 'up' if the upstream has been healthy long enough to not
+      // be a flapping signal — short blips are normal and we want the dashboard
+      // to be stable. After one full successful check we mark 'up' but the
+      // incident resolution waits for RESOLVED_AFTER_MS.
+      u.status = 'up';
+    } else {
+      u.consecutiveFailures += 1;
+      u.lastFailureAt = u.lastCheckedAt;
+      u.lastError = result.error || `HTTP ${result.statusCode || 'unknown'}`;
+      // First failure flips status to 'down' immediately for the dashboard, but
+      // we only OPEN an incident after the upstream has been continuously failing
+      // for DEAD_AFTER_MS (5 min by default) so a single transient blip doesn't
+      // page anyone.
+      u.status = 'down';
+      this._maybeOpenIncident(u);
+    }
+  }
+
+  _maybeOpenIncident(u) {
+    if (!this.healthChecker) return;
+    // "failingForMs" = continuous time the upstream has been unhealthy.
+    // Use lastSuccessAt as the anchor — if it was up 7min ago and is still
+    // down now, that's 7 minutes of continuous failure regardless of how many
+    // individual probe failures have piled up in between. Falls back to
+    // consecutiveFailures * interval when there's no success anchor (e.g. we've
+    // never seen the upstream healthy since startup).
+    const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
+    const failingForMs = lastSuccessMs !== null
+      ? Math.max(0, Date.now() - lastSuccessMs)
+      : u.consecutiveFailures * PROBE_INTERVAL_MS;
+    if (failingForMs < DEAD_AFTER_MS) return;
+    if (this.openIncidents.has(u.host)) return;
+
+    // Mimic the shape HealthChecker.createIncident expects.
+    try {
+      this.healthChecker.createIncident(u.host, 'caddy-upstream-dead',
+        `Caddy upstream ${u.host} (site ${u.site}) unreachable for ${Math.round(failingForMs / 60000)}m: ${u.lastError || 'no response'}`,
+        {
+          serviceId: u.host,
+          timestamp: u.lastFailureAt,
+          status: 'down',
+          error: u.lastError,
+          details: { site: u.site, siteFile: u.siteFile }
+        }
+      );
+      this.openIncidents.add(u.host);
+      this.emit('upstream-dead', u);
+      this.log.warn?.('caddy-upstream-watcher', `upstream dead: ${u.host} (${u.site})`);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `incident create failed: ${e.message}`);
+    }
+  }
+
+  _maybeResolve(u) {
+    if (!this.healthChecker) return;
+    if (!this.openIncidents.has(u.host)) return;
+    const downSince = u.lastFailureAt ? new Date(u.lastFailureAt).getTime() : 0;
+    const recoveredForMs = downSince ? Date.now() - downSince : 0;
+    if (recoveredForMs < RESOLVED_AFTER_MS) return;
+    try {
+      this.healthChecker.resolveIncident(u.host, 'caddy-upstream-dead', {
+        serviceId: u.host,
+        timestamp: u.lastSuccessAt || new Date().toISOString(),
+        status: 'up'
+      });
+      this.openIncidents.delete(u.host);
+      this.emit('upstream-recovered', u);
+      this.log.info?.('caddy-upstream-watcher', `upstream recovered: ${u.host}`);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `incident resolve failed: ${e.message}`);
+    }
+  }
+
+  _doProbe(host, port) {
+    return new Promise((resolve) => {
+      const isHttps = port === '443';
+      const lib = isHttps ? https : http;
+      const opts = {
+        hostname: host,
+        port: port || (isHttps ? 443 : 80),
+        method: 'HEAD',
+        path: '/',
+        timeout: PROBE_TIMEOUT_MS,
+        headers: { 'X-DashCaddy-HealthCheck': '1', 'User-Agent': 'DashCaddy-CaddyUpstreamWatcher/1' },
+        rejectUnauthorized: false
+      };
+      const req = lib.request(opts, (res) => {
+        res.resume();
+        const healthy = HEALTHY_CODES.has(res.statusCode);
+        resolve({ healthy, statusCode: res.statusCode });
+      });
+      req.on('timeout', () => {
+        req.destroy(new Error('probe timeout'));
+      });
+      req.on('error', (err) => {
+        resolve({ healthy: false, error: err.message });
+      });
+      req.end();
+    });
+  }
+
+  /** Public snapshot for the API/UI. */
+  snapshot() {
+    const list = [];
+    for (const u of this.upstreams.values()) {
+      const muted = this.muted.has(u.host);
+      // Same anchor as _maybeOpenIncident: time since the last successful
+      // probe. If we've never seen a success, fall back to consecutive
+      // failures × probe interval as a worst-case lower bound.
+      const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
+      let failingFor = 0;
+      if (!muted) {
+        if (lastSuccessMs !== null) {
+          failingFor = Math.max(0, Date.now() - lastSuccessMs);
+        } else if (u.status === 'down') {
+          failingFor = u.consecutiveFailures * PROBE_INTERVAL_MS;
+        }
+      }
+      list.push({
+        host: u.host,
+        site: u.site,
+        siteFile: u.siteFile,
+        status: muted ? 'muted' : u.status,
+        consecutiveFailures: u.consecutiveFailures,
+        lastCheckedAt: u.lastCheckedAt,
+        lastSuccessAt: u.lastSuccessAt,
+        lastFailureAt: u.lastFailureAt,
+        lastError: u.lastError,
+        failingForMs: failingFor,
+        muted,
+        dead: !muted && failingFor >= DEAD_AFTER_MS
+      });
+    }
+    // Sort: dead first, then down, then up, then unknown. Within each, by host.
+    list.sort((a, b) => {
+      const order = { dead: 0, down: 1, muted: 2, up: 3, unknown: 4 };
+      const oa = order[a.dead ? 'dead' : a.status] ?? 9;
+      const ob = order[b.dead ? 'dead' : b.status] ?? 9;
+      if (oa !== ob) return oa - ob;
+      return a.host.localeCompare(b.host);
+    });
+    return {
+      upstreams: list,
+      config: {
+        probeIntervalMs: PROBE_INTERVAL_MS,
+        deadAfterMs: DEAD_AFTER_MS,
+        resolvedAfterMs: RESOLVED_AFTER_MS,
+        sitesDir: SITES_DIR
+      }
+    };
+  }
+
+  setMuted(host, muted) {
+    if (muted) {
+      this.muted.add(host);
+    } else {
+      this.muted.delete(host);
+      // Reset failure state on unmute so we don't immediately re-incident a
+      // upstream that just came off mute.
+      const u = this.upstreams.get(host);
+      if (u) {
+        u.consecutiveFailures = 0;
+        u.lastError = null;
+        u.lastFailureAt = null;
+        u.status = 'unknown';
+      }
+    }
+    this._saveState();
+    return { host, muted: !!muted };
+  }
+
+  isMuted(host) { return this.muted.has(host); }
+
+  _loadState() {
+    try {
+      if (!fs.existsSync(STATE_FILE)) return;
+      const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
+      if (Array.isArray(data.muted)) this.muted = new Set(data.muted);
+      // Don't reload upstreams from disk — sites dir is the source of truth.
+      // But preserve last-check state for hosts that still exist.
+      if (data.upstreams && typeof data.upstreams === 'object') {
+        this._restoreUpstreamStates(data.upstreams);
+      }
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `state load failed: ${e.message}`);
+    }
+  }
+
+  _restoreUpstreamStates(persisted) {
+    for (const [host, st] of Object.entries(persisted)) {
+      if (this.upstreams.has(host)) continue;
+      this.upstreams.set(host, {
+        host,
+        ip: st.ip || host.split(':')[0],
+        port: st.port || null,
+        site: st.site || '',
+        siteFile: st.siteFile || '',
+        consecutiveFailures: st.consecutiveFailures || 0,
+        lastFailureAt: st.lastFailureAt || null,
+        lastSuccessAt: st.lastSuccessAt || null,
+        lastError: st.lastError || null,
+        lastCheckedAt: st.lastCheckedAt || null,
+        status: 'unknown'
+      });
+    }
+  }
+
+  _saveState() {
+    try {
+      const dir = path.dirname(STATE_FILE);
+      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+      const upstreams = {};
+      for (const [k, v] of this.upstreams.entries()) {
+        upstreams[k] = {
+          ip: v.ip,
+          port: v.port,
+          site: v.site,
+          siteFile: v.siteFile,
+          consecutiveFailures: v.consecutiveFailures,
+          lastFailureAt: v.lastFailureAt,
+          lastSuccessAt: v.lastSuccessAt,
+          lastError: v.lastError,
+          lastCheckedAt: v.lastCheckedAt
+        };
+      }
+      const tmp = STATE_FILE + '.tmp';
+      fs.writeFileSync(tmp, JSON.stringify({ muted: Array.from(this.muted), upstreams }, null, 2));
+      fs.renameSync(tmp, STATE_FILE);
+    } catch (e) {
+      this.log.warn?.('caddy-upstream-watcher', `state save failed: ${e.message}`);
+    }
+  }
+}
+
+// Singleton — matches the pattern of health-checker.js so it integrates
+// without a separate instantiation site.
+module.exports = new CaddyUpstreamWatcher();
+module.exports.CaddyUpstreamWatcher = CaddyUpstreamWatcher;