[glm-grade=B] fix(monitoring): remap loopback upstream probes to host gateway (DC-053)
The caddy-upstream-watcher runs inside the dashcaddy-api container but
probes upstreams declared for Caddy, which runs on the HOST. Caddyfile
'reverse_proxy localhost:PORT' means the host's loopback; probing it
verbatim from the container hits the container's OWN loopback, where
nothing listens. Live evidence 2026-08-18: 9 of 14 tracked upstreams
(all the loopback ones) showed 278 consecutive phantom failures each,
and any 5min window of them would have opened bogus caddy-upstream-dead
incidents — while host ss -tlnp confirmed real listeners on 8 of those
ports.
Fix:
- Probe loopback targets via host.docker.internal instead, pinned to the
host bridge IP by start.sh (--add-host=host.docker.internal:host-gateway,
Docker >= 20.10). Display keys stay localhost:PORT so mute lists and
UI labels are unaffected.
- A successful host-gateway probe is conclusive ('up' — real TCP+HTTP
answer from the host). A FAILED probe is epistemically inconclusive
(127.0.0.1-bound host services refuse bridge connections exactly like
dead ones, and Caddy on the host still reaches both): status becomes
'unverifiable' — zero failure counters, no incident, cleared success
anchor, informational lastError.
- IN_CONTAINER=false disables the remap (bare-metal deployments).
- Snapshot sort extended: dead > down > muted > unverifiable > up > unknown.
Tests: 5 new (23/23 in suite) covering remap targeting (localhost,
127.0.0.1, 127.x), non-loopback pass-through, unverifiable semantics,
and sort order. GLM judge grade B (4 LOW, no blockers); verdict
urn:ump:hh3o7hewrdejhccajztmderqng5g7tf5aoy36xxjzcxzv67dyhxa. Regrade
with Codex when quota resets 2026-08-24.
This commit is contained in:
@@ -379,11 +379,97 @@ describe('CaddyUpstreamWatcher', () => {
|
||||
fsState.exists[STATE] = true;
|
||||
// And the matching site file
|
||||
seedSites({ 'old.sami': 'old.sami { reverse_proxy 99.99.99.99:80 }\n' });
|
||||
|
||||
process.env.CADDY_UPSTREAMS_STATE_FILE = STATE;
|
||||
process.env.CADDY_SITES_DIR = SITES;
|
||||
const mod = require('../src/monitoring/caddy-upstream-watcher');
|
||||
const w = mod.CaddyUpstreamWatcher ? new mod.CaddyUpstreamWatcher() : mod;
|
||||
expect(w.isMuted('99.99.99.99:80')).toBe(true);
|
||||
});
|
||||
|
||||
// ---- Loopback → host-gateway remap (live bug 2026-08-18) -----------------
|
||||
// The watcher runs inside the container; Caddyfile `localhost:PORT` means
|
||||
// the HOST's loopback. Probing the container's own loopback gave 278
|
||||
// phantom failures per healthy host-side upstream.
|
||||
|
||||
test('loopback upstream probe is remapped to host.docker.internal (not container loopback)', async () => {
|
||||
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
||||
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
||||
const { w } = loadWatcher();
|
||||
await w.scanSites();
|
||||
const u = w.upstreams.get('localhost:8088');
|
||||
expect(u).toBeTruthy();
|
||||
const http = require('http');
|
||||
await w._probeOne(u);
|
||||
// The probe request must have gone to host.docker.internal, keeping the port.
|
||||
const call = http.request.mock.calls.find(c => c[0].hostname === 'host.docker.internal');
|
||||
expect(call).toBeTruthy();
|
||||
expect(call[0].port).toBe('8088');
|
||||
// Display key is unchanged.
|
||||
expect(w.snapshot().upstreams[0].host).toBe('localhost:8088');
|
||||
expect(w.snapshot().upstreams[0].status).toBe('up');
|
||||
});
|
||||
|
||||
test('127.0.0.1 and 127.x addresses are also remapped', async () => {
|
||||
seedSites({
|
||||
'a.sami': 'a.sami { reverse_proxy 127.0.0.1:8765 }\n',
|
||||
'b.sami': 'b.sami { reverse_proxy 127.0.0.53:9000 }\n'
|
||||
});
|
||||
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
||||
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
||||
const { w } = loadWatcher();
|
||||
await w.scanSites();
|
||||
const http = require('http');
|
||||
for (const u of w.upstreams.values()) await w._probeOne(u);
|
||||
const hostnames = http.request.mock.calls.map(c => c[0].hostname);
|
||||
expect(hostnames).toEqual(['host.docker.internal', 'host.docker.internal']);
|
||||
});
|
||||
|
||||
test('non-loopback upstreams are probed at their literal address (no remap)', async () => {
|
||||
seedSites({ 'arch.sami': 'arch.sami { reverse_proxy 100.120.159.34:5000 }\n' });
|
||||
probeQueue.push({ kind: 'ok', statusCode: 200 });
|
||||
const { w } = loadWatcher();
|
||||
await w.scanSites();
|
||||
const http = require('http');
|
||||
await w._probeOne(w.upstreams.get('100.120.159.34:5000'));
|
||||
const hostnames = http.request.mock.calls.map(c => c[0].hostname);
|
||||
expect(hostnames).toEqual(['100.120.159.34']);
|
||||
});
|
||||
|
||||
test('failed host-gateway probe marks loopback upstream unverifiable — no failures, no incident', async () => {
|
||||
// Services bound to 127.0.0.1 on the host refuse bridge-IP connections;
|
||||
// from inside the container that is indistinguishable from "dead", and
|
||||
// Caddy (on the host) still routes fine — so it must NOT count as down.
|
||||
seedSites({ 'zap.sami': 'zap.sami { reverse_proxy localhost:8088 }\n' });
|
||||
probeQueue.push({ kind: 'err', message: 'connect ECONNREFUSED 172.17.0.1:8088' });
|
||||
const { w } = loadWatcher();
|
||||
const fakeHealthChecker = { createIncident: jest.fn(), resolveIncident: jest.fn(), incidents: [] };
|
||||
w.healthChecker = fakeHealthChecker;
|
||||
await w.scanSites();
|
||||
const u = w.upstreams.get('localhost:8088');
|
||||
u.lastSuccessAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); // long "failing" history
|
||||
await w._probeOne(u);
|
||||
const snap = w.snapshot().upstreams[0];
|
||||
expect(snap.status).toBe('unverifiable');
|
||||
expect(snap.consecutiveFailures).toBe(0);
|
||||
expect(snap.dead).toBe(false);
|
||||
expect(snap.failingForMs).toBe(0);
|
||||
expect(snap.lastError).toMatch(/not verifiable from container/);
|
||||
expect(fakeHealthChecker.createIncident).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('unverifiable sorts between muted and up in the snapshot', async () => {
|
||||
seedSites({
|
||||
'a.sami': 'a.sami { reverse_proxy 1.1.1.1:80 }\n',
|
||||
'b.sami': 'b.sami { reverse_proxy localhost:9876 }\n',
|
||||
'c.sami': 'c.sami { reverse_proxy 2.2.2.2:80 }\n'
|
||||
});
|
||||
const { w } = loadWatcher();
|
||||
await w.scanSites();
|
||||
const all = Array.from(w.upstreams.values());
|
||||
all.find(u => u.host === '1.1.1.1:80').status = 'up';
|
||||
all.find(u => u.host === 'localhost:9876').status = 'unverifiable';
|
||||
w.muted.add('2.2.2.2:80');
|
||||
const order = w.snapshot().upstreams.map(u => u.host);
|
||||
expect(order).toEqual(['2.2.2.2:80', 'localhost:9876', '1.1.1.1:80']);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user