[glm-grade=B] fix(monitoring): remap loopback upstream probes to host gateway (DC-053)

The caddy-upstream-watcher runs inside the dashcaddy-api container but
probes upstreams declared for Caddy, which runs on the HOST. Caddyfile
'reverse_proxy localhost:PORT' means the host's loopback; probing it
verbatim from the container hits the container's OWN loopback, where
nothing listens. Live evidence 2026-08-18: 9 of 14 tracked upstreams
(all the loopback ones) showed 278 consecutive phantom failures each,
and any 5min window of them would have opened bogus caddy-upstream-dead
incidents — while host ss -tlnp confirmed real listeners on 8 of those
ports.

Fix:
- Probe loopback targets via host.docker.internal instead, pinned to the
  host bridge IP by start.sh (--add-host=host.docker.internal:host-gateway,
  Docker >= 20.10). Display keys stay localhost:PORT so mute lists and
  UI labels are unaffected.
- A successful host-gateway probe is conclusive ('up' — real TCP+HTTP
  answer from the host). A FAILED probe is epistemically inconclusive
  (127.0.0.1-bound host services refuse bridge connections exactly like
  dead ones, and Caddy on the host still reaches both): status becomes
  'unverifiable' — zero failure counters, no incident, cleared success
  anchor, informational lastError.
- IN_CONTAINER=false disables the remap (bare-metal deployments).
- Snapshot sort extended: dead > down > muted > unverifiable > up > unknown.

Tests: 5 new (23/23 in suite) covering remap targeting (localhost,
127.0.0.1, 127.x), non-loopback pass-through, unverifiable semantics,
and sort order. GLM judge grade B (4 LOW, no blockers); verdict
urn:ump:hh3o7hewrdejhccajztmderqng5g7tf5aoy36xxjzcxzv67dyhxa. Regrade
with Codex when quota resets 2026-08-24.

dashcaddy-api/__tests__/caddy-upstream-watcher.test.js

@@ -379,11 +379,97 @@ describe('CaddyUpstreamWatcher', () => {
     fsState.exists[STATE] = true;
     // And the matching site file
     seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' });
-
     process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
     process.env.CADDY_SITES_DIR = SITES;
     const mod = require('../src/monitoring/caddy-upstream-watcher');
     const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod;
     expect(w.isMuted('99.99.99.99:80')).toBe(true);
   });
+
+  // ---- Loopback → host-gateway remap (live bug 2026-08-18) -----------------
+  // The watcher runs inside the container; Caddyfile `localhost:PORT` means
+  // the HOST's loopback. Probing the container's own loopback gave 278
+  // phantom failures per healthy host-side upstream.
+
+  test('loopback upstream probe is remapped to host.docker.internal (not container loopback)', async () => {
+    seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
+    probeQueue.push({ kind: 'ok', statusCode: 200 });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const u = w.upstreams.get('localhost:8088');
+    expect(u).toBeTruthy();
+    const http = require('http');
+    await w._probeOne(u);
+    // The probe request must have gone to host.docker.internal, keeping the port.
+    const call = http.request.mock.calls.find(c => c[0].hostname === 'host.docker.internal');
+    expect(call).toBeTruthy();
+    expect(call[0].port).toBe('8088');
+    // Display key is unchanged.
+    expect(w.snapshot().upstreams[0].host).toBe('localhost:8088');
+    expect(w.snapshot().upstreams[0].status).toBe('up');
+  });
+
+  test('127.0.0.1 and 127.x addresses are also remapped', async () => {
+    seedSites({
+      'a.sami': 'a.sami { reverse_proxy 127.0.0.1:8765 }\n',
+      'b.sami': 'b.sami { reverse_proxy 127.0.0.53:9000 }\n'
+    });
+    probeQueue.push({ kind: 'ok', statusCode: 200 });
+    probeQueue.push({ kind: 'ok', statusCode: 200 });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const http = require('http');
+    for (const u of w.upstreams.values()) await w._probeOne(u);
+    const hostnames = http.request.mock.calls.map(c => c[0].hostname);
+    expect(hostnames).toEqual(['host.docker.internal', 'host.docker.internal']);
+  });
+
+  test('non-loopback upstreams are probed at their literal address (no remap)', async () => {
+    seedSites({ 'arch.sami': 'arch.sami { reverse_proxy 100.120.159.34:5000 }\n' });
+    probeQueue.push({ kind: 'ok', statusCode: 200 });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const http = require('http');
+    await w._probeOne(w.upstreams.get('100.120.159.34:5000'));
+    const hostnames = http.request.mock.calls.map(c => c[0].hostname);
+    expect(hostnames).toEqual(['100.120.159.34']);
+  });
+
+  test('failed host-gateway probe marks loopback upstream unverifiable — no failures, no incident', async () => {
+    // Services bound to 127.0.0.1 on the host refuse bridge-IP connections;
+    // from inside the container that is indistinguishable from "dead", and
+    // Caddy (on the host) still routes fine — so it must NOT count as down.
+    seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
+    probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED 172.17.0.1:8088' });
+    const { w } = loadWatcher();
+    const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] };
+    w.healthChecker = fakeHealthChecker;
+    await w.scanSites();
+    const u = w.upstreams.get('localhost:8088');
+    u.lastSuccessAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); // long "failing" history
+    await w._probeOne(u);
+    const snap = w.snapshot().upstreams[0];
+    expect(snap.status).toBe('unverifiable');
+    expect(snap.consecutiveFailures).toBe(0);
+    expect(snap.dead).toBe(false);
+    expect(snap.failingForMs).toBe(0);
+    expect(snap.lastError).toMatch(/not verifiable from container/);
+    expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
+  });
+
+  test('unverifiable sorts between muted and up in the snapshot', async () => {
+    seedSites({
+      'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
+      'b.sami': 'b.sami { reverse_proxy localhost:9876 }\n',
+      'c.sami': 'c.sami { reverse_proxy 2.2.2.2:80 }\n'
+    });
+    const { w } = loadWatcher();
+    await w.scanSites();
+    const all = Array.from(w.upstreams.values());
+    all.find(u => u.host === '1.1.1.1:80').status = 'up';
+    all.find(u => u.host === 'localhost:9876').status = 'unverifiable';
+    w.muted.add('2.2.2.2:80');
+    const order = w.snapshot().upstreams.map(u => u.host);
+    expect(order).toEqual(['2.2.2.2:80', 'localhost:9876', '1.1.1.1:80']);
+  });
 });

dashcaddy-api/src/monitoring/caddy-upstream-watcher.js

@@ -52,6 +52,34 @@ const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE
 
 const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites';
 
+/**
+ * Hostname the probe uses instead of a loopback address.
+ *
+ * CRITICAL: this watcher runs INSIDE the dashcaddy-api container. Caddy runs
+ * on the HOST. A site config's `reverse_proxy localhost:8088` means "the
+ * host's loopback" from Caddy's point of view — but from inside the container
+ * `localhost`/`127.0.0.1` is the container's OWN loopback, where nothing
+ * listens. Probing loopback verbatim makes every healthy host-side upstream
+ * report ECONNREFUSED (live prod bug 2026-08-18: 9 of 14 tracked upstreams
+ * showed 278 consecutive phantom failures and opened bogus `caddy-upstream-dead`
+ * incidents).
+ *
+ * Fix: remap loopback probe targets to `host.docker.internal`, which start.sh
+ * pins to the host's bridge IP via `--add-host=host.docker.internal:host-gateway`
+ * (Docker ≥ 20.10). The upstream's display key stays `localhost:PORT` so
+ * existing mute lists and UI labels are unaffected — only the probe target
+ * changes. Set IN_CONTAINER=false (e.g. a bare-metal deployment where the API
+ * runs beside Caddy) to disable the remap.
+ */
+const HOST_GATEWAY_NAME = process.env.CADDY_UPSTREAM_HOST_GATEWAY_NAME || 'host.docker.internal';
+const IN_CONTAINER = process.env.IN_CONTAINER !== 'false';
+const HOST_GATEWAY_PROBE = IN_CONTAINER ? HOST_GATEWAY_NAME : null;
+
+/** True when the address is IPv4 loopback (127.0.0.0/8) or the `localhost` name. */
+function isLoopbackHost(host) {
+  return host === 'localhost' || /^127(\.\d{1,3}){3}$/.test(host);
+}
+
 class CaddyUpstreamWatcher extends EventEmitter {
   constructor(opts = {}) {
     super();
@@ -195,7 +223,12 @@ class CaddyUpstreamWatcher extends EventEmitter {
 
   /** Probe a single upstream and update state. */
   async _probeOne(u) {
-    const result = await this._doProbe(u.ip, u.port);
+    // Loopback upstreams (see HOST_GATEWAY_PROBE header comment): the Caddyfile
+    // `localhost`/`127.x` is host-relative, so probe the host gateway instead of
+    // the container's own loopback. Display key and persisted `ip` are unchanged.
+    const loopbackRemap = !!(HOST_GATEWAY_PROBE && isLoopbackHost(u.ip));
+    const probeHost = loopbackRemap ? HOST_GATEWAY_PROBE : u.ip;
+    const result = await this._doProbe(probeHost, u.port);
     u.lastCheckedAt = new Date().toISOString();
 
     if (result.healthy) {
@@ -209,6 +242,23 @@ class CaddyUpstreamWatcher extends EventEmitter {
       // to be stable. After one full successful check we mark 'up' but the
       // incident resolution waits for RESOLVED_AFTER_MS.
       u.status = 'up';
+    } else if (loopbackRemap) {
+      // The host-gateway probe comes from the docker bridge IP. A service
+      // bound to 0.0.0.0 on the host answers; a service bound to the host's
+      // 127.0.0.1 ONLY refuses — indistinguishable, from this vantage point,
+      // from a truly dead service. Caddy (on the host) reaches both fine, so
+      // a failed probe here is NOT evidence the upstream is dead. Mark it
+      // unverifiable: no failure counters, no incident, keep lastError for
+      // visibility. (A successful probe IS conclusive — see above.)
+      u.consecutiveFailures = 0;
+      u.status = 'unverifiable';
+      u.lastError = `host-loopback upstream not verifiable from container (${result.error || `HTTP ${result.statusCode || 'unknown'}`})`;
+      // Clear the success anchor: a 10-minute-old success is not evidence of
+      // anything for an upstream we cannot observe from this vantage point,
+      // and leaving it would make snapshot() compute a bogus failingForMs
+      // and flag `dead`.
+      u.lastSuccessAt = null;
+      this._maybeResolve(u);
     } else {
       u.consecutiveFailures += 1;
       u.lastFailureAt = u.lastCheckedAt;
@@ -337,9 +387,10 @@ class CaddyUpstreamWatcher extends EventEmitter {
         dead: !muted && failingFor >= DEAD_AFTER_MS
       });
     }
-    // Sort: dead first, then down, then up, then unknown. Within each, by host.
+    // Sort: dead first, then down, then muted, then unverifiable (informational),
+    // then up, then unknown. Within each, by host.
     list.sort((a, b) => {
-      const order = { dead: 0, down: 1, muted: 2, up: 3, unknown: 4 };
+      const order = { dead: 0, down: 1, muted: 2, unverifiable: 3, up: 4, unknown: 5 };
       const oa = order[a.dead ? 'dead' : a.status] ?? 9;
       const ob = order[b.dead ? 'dead' : b.status] ?? 9;
       if (oa !== ob) return oa - ob;

start.sh

@@ -88,6 +88,14 @@ run_image_layer_migration
 #   git.dashcaddy.net → DashCaddy upstream git
 #   ca.sami           → local CA (DN2 + DN3 both have their own)
 ADD_HOST_FLAGS=(
+  # host.docker.internal → host bridge IP (Docker host-gateway). The caddy
+  # upstream watcher probes Caddy site upstreams from INSIDE this container;
+  # `reverse_proxy localhost:PORT` in a site file means the HOST's loopback,
+  # so the watcher remaps loopback probe targets to this name (see
+  # dashcaddy-api/src/monitoring/caddy-upstream-watcher.js). Without this
+  # entry the probes would hit the container's own loopback and report every
+  # host-side upstream as dead.
+  --add-host=host.docker.internal:host-gateway
   --add-host=dns3.sami:100.81.59.99
   --add-host=gitea:100.81.59.99
   --add-host=dns3-wan.sami:74.208.167.19