From 71e04d0a86e69bb844f6dbd05ca9878da4553c3c Mon Sep 17 00:00:00 2001 From: Hermes Date: Mon, 17 Aug 2026 22:46:32 -0700 Subject: [PATCH] [glm-grade=B] fix(monitoring): remap loopback upstream probes to host gateway (DC-053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caddy-upstream-watcher runs inside the dashcaddy-api container but probes upstreams declared for Caddy, which runs on the HOST. Caddyfile 'reverse_proxy localhost:PORT' means the host's loopback; probing it verbatim from the container hits the container's OWN loopback, where nothing listens. Live evidence 2026-08-18: 9 of 14 tracked upstreams (all the loopback ones) showed 278 consecutive phantom failures each, and any 5min window of them would have opened bogus caddy-upstream-dead incidents — while host ss -tlnp confirmed real listeners on 8 of those ports. Fix: - Probe loopback targets via host.docker.internal instead, pinned to the host bridge IP by start.sh (--add-host=host.docker.internal:host-gateway, Docker >= 20.10). Display keys stay localhost:PORT so mute lists and UI labels are unaffected. - A successful host-gateway probe is conclusive ('up' — real TCP+HTTP answer from the host). A FAILED probe is epistemically inconclusive (127.0.0.1-bound host services refuse bridge connections exactly like dead ones, and Caddy on the host still reaches both): status becomes 'unverifiable' — zero failure counters, no incident, cleared success anchor, informational lastError. - IN_CONTAINER=false disables the remap (bare-metal deployments). - Snapshot sort extended: dead > down > muted > unverifiable > up > unknown. Tests: 5 new (23/23 in suite) covering remap targeting (localhost, 127.0.0.1, 127.x), non-loopback pass-through, unverifiable semantics, and sort order. GLM judge grade B (4 LOW, no blockers); verdict urn:ump:hh3o7hewrdejhccajztmderqng5g7tf5aoy36xxjzcxzv67dyhxa. Regrade with Codex when quota resets 2026-08-24. --- .../__tests__/caddy-upstream-watcher.test.js | 88 ++++++++++++++++++- .../src/monitoring/caddy-upstream-watcher.js | 57 +++++++++++- start.sh | 8 ++ 3 files changed, 149 insertions(+), 4 deletions(-) diff --git a/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js b/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js index 04228ef..8134d4d 100644 --- a/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js +++ b/dashcaddy-api/__tests__/caddy-upstream-watcher.test.js @@ -379,11 +379,97 @@ describe('CaddyUpstreamWatcher', () => { fsState.exists[STATE] = true; // And the matching site file seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' }); - process.env.CADDY_UPSTREAMS_STATE_FILE = STATE; process.env.CADDY_SITES_DIR = SITES; const mod = require('../src/monitoring/caddy-upstream-watcher'); const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod; expect(w.isMuted('99.99.99.99:80')).toBe(true); }); + + // ---- Loopback → host-gateway remap (live bug 2026-08-18) ----------------- + // The watcher runs inside the container; Caddyfile `localhost:PORT` means + // the HOST's loopback. Probing the container's own loopback gave 278 + // phantom failures per healthy host-side upstream. + + test('loopback upstream probe is remapped to host.docker.internal (not container loopback)', async () => { + seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' }); + probeQueue.push({ kind: 'ok', statusCode: 200 }); + const { w } = loadWatcher(); + await w.scanSites(); + const u = w.upstreams.get('localhost:8088'); + expect(u).toBeTruthy(); + const http = require('http'); + await w._probeOne(u); + // The probe request must have gone to host.docker.internal, keeping the port. + const call = http.request.mock.calls.find(c => c[0].hostname === 'host.docker.internal'); + expect(call).toBeTruthy(); + expect(call[0].port).toBe('8088'); + // Display key is unchanged. + expect(w.snapshot().upstreams[0].host).toBe('localhost:8088'); + expect(w.snapshot().upstreams[0].status).toBe('up'); + }); + + test('127.0.0.1 and 127.x addresses are also remapped', async () => { + seedSites({ + 'a.sami': 'a.sami { reverse_proxy 127.0.0.1:8765 }\n', + 'b.sami': 'b.sami { reverse_proxy 127.0.0.53:9000 }\n' + }); + probeQueue.push({ kind: 'ok', statusCode: 200 }); + probeQueue.push({ kind: 'ok', statusCode: 200 }); + const { w } = loadWatcher(); + await w.scanSites(); + const http = require('http'); + for (const u of w.upstreams.values()) await w._probeOne(u); + const hostnames = http.request.mock.calls.map(c => c[0].hostname); + expect(hostnames).toEqual(['host.docker.internal', 'host.docker.internal']); + }); + + test('non-loopback upstreams are probed at their literal address (no remap)', async () => { + seedSites({ 'arch.sami': 'arch.sami { reverse_proxy 100.120.159.34:5000 }\n' }); + probeQueue.push({ kind: 'ok', statusCode: 200 }); + const { w } = loadWatcher(); + await w.scanSites(); + const http = require('http'); + await w._probeOne(w.upstreams.get('100.120.159.34:5000')); + const hostnames = http.request.mock.calls.map(c => c[0].hostname); + expect(hostnames).toEqual(['100.120.159.34']); + }); + + test('failed host-gateway probe marks loopback upstream unverifiable — no failures, no incident', async () => { + // Services bound to 127.0.0.1 on the host refuse bridge-IP connections; + // from inside the container that is indistinguishable from "dead", and + // Caddy (on the host) still routes fine — so it must NOT count as down. + seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' }); + probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED 172.17.0.1:8088' }); + const { w } = loadWatcher(); + const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] }; + w.healthChecker = fakeHealthChecker; + await w.scanSites(); + const u = w.upstreams.get('localhost:8088'); + u.lastSuccessAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); // long "failing" history + await w._probeOne(u); + const snap = w.snapshot().upstreams[0]; + expect(snap.status).toBe('unverifiable'); + expect(snap.consecutiveFailures).toBe(0); + expect(snap.dead).toBe(false); + expect(snap.failingForMs).toBe(0); + expect(snap.lastError).toMatch(/not verifiable from container/); + expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled(); + }); + + test('unverifiable sorts between muted and up in the snapshot', async () => { + seedSites({ + 'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n', + 'b.sami': 'b.sami { reverse_proxy localhost:9876 }\n', + 'c.sami': 'c.sami { reverse_proxy 2.2.2.2:80 }\n' + }); + const { w } = loadWatcher(); + await w.scanSites(); + const all = Array.from(w.upstreams.values()); + all.find(u => u.host === '1.1.1.1:80').status = 'up'; + all.find(u => u.host === 'localhost:9876').status = 'unverifiable'; + w.muted.add('2.2.2.2:80'); + const order = w.snapshot().upstreams.map(u => u.host); + expect(order).toEqual(['2.2.2.2:80', 'localhost:9876', '1.1.1.1:80']); + }); }); \ No newline at end of file diff --git a/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js b/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js index 93ffc4c..c7c996e 100644 --- a/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js +++ b/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js @@ -52,6 +52,34 @@ const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites'; +/** + * Hostname the probe uses instead of a loopback address. + * + * CRITICAL: this watcher runs INSIDE the dashcaddy-api container. Caddy runs + * on the HOST. A site config's `reverse_proxy localhost:8088` means "the + * host's loopback" from Caddy's point of view — but from inside the container + * `localhost`/`127.0.0.1` is the container's OWN loopback, where nothing + * listens. Probing loopback verbatim makes every healthy host-side upstream + * report ECONNREFUSED (live prod bug 2026-08-18: 9 of 14 tracked upstreams + * showed 278 consecutive phantom failures and opened bogus `caddy-upstream-dead` + * incidents). + * + * Fix: remap loopback probe targets to `host.docker.internal`, which start.sh + * pins to the host's bridge IP via `--add-host=host.docker.internal:host-gateway` + * (Docker ≥ 20.10). The upstream's display key stays `localhost:PORT` so + * existing mute lists and UI labels are unaffected — only the probe target + * changes. Set IN_CONTAINER=false (e.g. a bare-metal deployment where the API + * runs beside Caddy) to disable the remap. + */ +const HOST_GATEWAY_NAME = process.env.CADDY_UPSTREAM_HOST_GATEWAY_NAME || 'host.docker.internal'; +const IN_CONTAINER = process.env.IN_CONTAINER !== 'false'; +const HOST_GATEWAY_PROBE = IN_CONTAINER ? HOST_GATEWAY_NAME : null; + +/** True when the address is IPv4 loopback (127.0.0.0/8) or the `localhost` name. */ +function isLoopbackHost(host) { + return host === 'localhost' || /^127(\.\d{1,3}){3}$/.test(host); +} + class CaddyUpstreamWatcher extends EventEmitter { constructor(opts = {}) { super(); @@ -195,7 +223,12 @@ class CaddyUpstreamWatcher extends EventEmitter { /** Probe a single upstream and update state. */ async _probeOne(u) { - const result = await this._doProbe(u.ip, u.port); + // Loopback upstreams (see HOST_GATEWAY_PROBE header comment): the Caddyfile + // `localhost`/`127.x` is host-relative, so probe the host gateway instead of + // the container's own loopback. Display key and persisted `ip` are unchanged. + const loopbackRemap = !!(HOST_GATEWAY_PROBE && isLoopbackHost(u.ip)); + const probeHost = loopbackRemap ? HOST_GATEWAY_PROBE : u.ip; + const result = await this._doProbe(probeHost, u.port); u.lastCheckedAt = new Date().toISOString(); if (result.healthy) { @@ -209,6 +242,23 @@ class CaddyUpstreamWatcher extends EventEmitter { // to be stable. After one full successful check we mark 'up' but the // incident resolution waits for RESOLVED_AFTER_MS. u.status = 'up'; + } else if (loopbackRemap) { + // The host-gateway probe comes from the docker bridge IP. A service + // bound to 0.0.0.0 on the host answers; a service bound to the host's + // 127.0.0.1 ONLY refuses — indistinguishable, from this vantage point, + // from a truly dead service. Caddy (on the host) reaches both fine, so + // a failed probe here is NOT evidence the upstream is dead. Mark it + // unverifiable: no failure counters, no incident, keep lastError for + // visibility. (A successful probe IS conclusive — see above.) + u.consecutiveFailures = 0; + u.status = 'unverifiable'; + u.lastError = `host-loopback upstream not verifiable from container (${result.error || `HTTP ${result.statusCode || 'unknown'}`})`; + // Clear the success anchor: a 10-minute-old success is not evidence of + // anything for an upstream we cannot observe from this vantage point, + // and leaving it would make snapshot() compute a bogus failingForMs + // and flag `dead`. + u.lastSuccessAt = null; + this._maybeResolve(u); } else { u.consecutiveFailures += 1; u.lastFailureAt = u.lastCheckedAt; @@ -337,9 +387,10 @@ class CaddyUpstreamWatcher extends EventEmitter { dead: !muted && failingFor >= DEAD_AFTER_MS }); } - // Sort: dead first, then down, then up, then unknown. Within each, by host. + // Sort: dead first, then down, then muted, then unverifiable (informational), + // then up, then unknown. Within each, by host. list.sort((a, b) => { - const order = { dead: 0, down: 1, muted: 2, up: 3, unknown: 4 }; + const order = { dead: 0, down: 1, muted: 2, unverifiable: 3, up: 4, unknown: 5 }; const oa = order[a.dead ? 'dead' : a.status] ?? 9; const ob = order[b.dead ? 'dead' : b.status] ?? 9; if (oa !== ob) return oa - ob; diff --git a/start.sh b/start.sh index bf06994..a68e5c1 100755 --- a/start.sh +++ b/start.sh @@ -88,6 +88,14 @@ run_image_layer_migration # git.dashcaddy.net → DashCaddy upstream git # ca.sami → local CA (DN2 + DN3 both have their own) ADD_HOST_FLAGS=( + # host.docker.internal → host bridge IP (Docker host-gateway). The caddy + # upstream watcher probes Caddy site upstreams from INSIDE this container; + # `reverse_proxy localhost:PORT` in a site file means the HOST's loopback, + # so the watcher remaps loopback probe targets to this name (see + # dashcaddy-api/src/monitoring/caddy-upstream-watcher.js). Without this + # entry the probes would hit the container's own loopback and report every + # host-side upstream as dead. + --add-host=host.docker.internal:host-gateway --add-host=dns3.sami:100.81.59.99 --add-host=gitea:100.81.59.99 --add-host=dns3-wan.sami:74.208.167.19