From ba23cdff028b3821a0a58a91bc7a707b30846c0d Mon Sep 17 00:00:00 2001 From: Krystie Date: Sun, 5 Jul 2026 12:46:35 -0700 Subject: [PATCH] DC-032: fix health checker authLimiter feedback loop + ca.sami DNS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three coordinated changes to stop every gated *.sami service flipping red after ~20 probes: 1. health-checker.js _doRequest() now sends X-DashCaddy-HealthCheck: 1 on every outgoing probe. Caddy uses this header (combined with a trusted source IP via the new @healthcheckProbe matcher in the dashcaddy_auth snippet) to bypass forward_auth for local container probes. Without the bypass, forward_auth 401's every probe, and the authLimiter (20 req / 15 min, DC-027) caps us out within minutes. 2. evaluateHealth() default expectedStatusCodes now includes 401, 403, and 429. Defense in depth — if a future Caddy reload drops the bypass, 429 from the rate-limited gate no longer marks the service as down (it just means the gate answered, which proves the service is reachable through Caddy). 3. (start.sh — already shipped on the running container, will land with the next release build) ca.sami now maps to 100.121.150.22 (DNS2) instead of 127.0.0.1, which is the container's own loopback where nothing serves :443. The CA web UI lives on DNS2's Caddy. Tests: - evaluateHealth: 401, 403, 429 accepted by default - _doRequest: X-DashCaddy-HealthCheck: 1 always present - _doRequest: user-supplied headers preserved alongside marker Bump 1.14.7 → 1.14.8. --- VERSION | 2 +- .../__tests__/health-checker.test.js | 89 +++++++++++++++++++ dashcaddy-api/package.json | 2 +- .../src/monitoring/health-checker.js | 22 ++++- 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 52e779f..9be7846 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.7 +1.14.8 diff --git a/dashcaddy-api/__tests__/health-checker.test.js b/dashcaddy-api/__tests__/health-checker.test.js index 3b0fc24..55f2e74 100644 --- a/dashcaddy-api/__tests__/health-checker.test.js +++ b/dashcaddy-api/__tests__/health-checker.test.js @@ -125,6 +125,18 @@ describe('HealthChecker', () => { expect(healthChecker.evaluateHealth(500, '', {})).toBe(false); }); + it('defaults to accepting 401/403 (auth-walled UIs still prove the service is up)', () => { + expect(healthChecker.evaluateHealth(401, '', {})).toBe(true); + expect(healthChecker.evaluateHealth(403, '', {})).toBe(true); + }); + + it('defaults to accepting 429 (rate-limited upstream is still reachable)', () => { + // The upstream answered — it just throttled us. Failing the check here + // caused the authLimiter feedback loop (DC-XXX) where every gated + // service flipped red after 20 probes / 15 min. + expect(healthChecker.evaluateHealth(429, '', {})).toBe(true); + }); + it('checks body pattern with regex', () => { const config = { expectedBodyPattern: 'ok|healthy' }; expect(healthChecker.evaluateHealth(200, 'status: ok', config)).toBe(true); @@ -241,6 +253,83 @@ describe('HealthChecker', () => { }); }); + describe('_doRequest header injection', () => { + // Verifies the X-DashCaddy-HealthCheck marker header is set on every + // outgoing probe. Caddy uses this header (combined with a trusted source + // IP) to bypass forward_auth for probes from the local container, which + // is what stops the authLimiter feedback loop on gated services. + // CI doesn't make real network calls — we capture the options object + // via a tiny http mock and assert on it. + // + // Note: the suite runs under jest.useFakeTimers(), so we cannot rely on + // setImmediate / setTimeout to fire the fake response. We emit 'end' + // synchronously after attaching listeners, which the response handler + // in _doRequest will receive on the same tick. + it('sends X-DashCaddy-HealthCheck: 1 on every probe', () => { + const https = require('https'); + const { EventEmitter } = require('events'); + const original = https.request; + let capturedOptions = null; + https.request = (options, cb) => { + capturedOptions = options; + const fakeRes = new EventEmitter(); + fakeRes.statusCode = 200; + fakeRes.headers = {}; + // Call cb synchronously so listeners attach BEFORE we emit 'end'. + cb(fakeRes); + fakeRes.emit('end'); + const fakeReq = new EventEmitter(); + fakeReq.end = () => {}; + fakeReq.write = () => {}; + fakeReq.destroy = () => {}; + return fakeReq; + }; + + try { + return healthChecker._doRequest({ url: 'https://example.sami/test', method: 'HEAD' }, 'HEAD').then(() => { + expect(capturedOptions).not.toBeNull(); + expect(capturedOptions.headers['X-DashCaddy-HealthCheck']).toBe('1'); + }); + } finally { + https.request = original; + } + }); + + it('preserves user-supplied headers while adding the marker', () => { + const https = require('https'); + const { EventEmitter } = require('events'); + const original = https.request; + let capturedOptions = null; + https.request = (options, cb) => { + capturedOptions = options; + const fakeRes = new EventEmitter(); + fakeRes.statusCode = 200; + fakeRes.headers = {}; + cb(fakeRes); + fakeRes.emit('end'); + const fakeReq = new EventEmitter(); + fakeReq.end = () => {}; + fakeReq.write = () => {}; + fakeReq.destroy = () => {}; + return fakeReq; + }; + + try { + return healthChecker._doRequest({ + url: 'https://example.sami/test', + method: 'GET', + headers: { 'User-Agent': 'DashCaddy-Test/1.0', 'X-Custom': 'foo' } + }, 'GET').then(() => { + expect(capturedOptions.headers['X-DashCaddy-HealthCheck']).toBe('1'); + expect(capturedOptions.headers['User-Agent']).toBe('DashCaddy-Test/1.0'); + expect(capturedOptions.headers['X-Custom']).toBe('foo'); + }); + } finally { + https.request = original; + } + }); + }); + describe('incidents', () => { it('createIncident adds a new incident', () => { const status = { timestamp: new Date().toISOString() }; diff --git a/dashcaddy-api/package.json b/dashcaddy-api/package.json index 5479470..b63f2f8 100644 --- a/dashcaddy-api/package.json +++ b/dashcaddy-api/package.json @@ -1,6 +1,6 @@ { "name": "dashcaddy-api", - "version": "1.14.7", + "version": "1.14.8", "description": "DashCaddy API server - Dashboard backend for Docker, Caddy & DNS management", "main": "server.js", "scripts": { diff --git a/dashcaddy-api/src/monitoring/health-checker.js b/dashcaddy-api/src/monitoring/health-checker.js index e458c59..2b3841f 100644 --- a/dashcaddy-api/src/monitoring/health-checker.js +++ b/dashcaddy-api/src/monitoring/health-checker.js @@ -178,13 +178,26 @@ class HealthChecker extends EventEmitter { const url = new URL(config.url); const protocol = url.protocol === 'https:' ? https : http; + // Merge user-supplied headers with the health-check marker. Caddy on + // *.sami uses `forward_auth` for every non-API path and the auth gate + // returns 401 for HEAD/GET without a session — without this marker the + // probe never reaches the upstream service, and the authLimiter (20 req + // / 15 min) on /auth/* would also rate-limit us after 20 probes. The + // marker lets the Caddy snippet bypass forward_auth for probes that + // originate from the local container network (see /etc/caddy/Caddyfile + // `(dashcaddy_auth)` block). + const headers = { + ...(config.headers || {}), + 'X-DashCaddy-HealthCheck': '1' + }; + const options = { hostname: url.hostname, port: url.port || (url.protocol === 'https:' ? 443 : 80), path: url.pathname + url.search, method, timeout: config.timeout || 20000, - headers: config.headers || {}, + headers, rejectUnauthorized: false // Trust internal CA certs (.sami TLD) }; @@ -231,8 +244,11 @@ class HealthChecker extends EventEmitter { * Evaluate if service is healthy based on response */ evaluateHealth(statusCode, body, config) { - // Check status code - const expectedCodes = config.expectedStatusCodes || [200, 201, 204, 301, 302, 303, 307, 308]; + // Check status code. Default expected codes include the usual 2xx/3xx + // plus 401/403 (auth-walled UIs that still prove the service is up) and + // 429 (rate-limited upstream — we hit the service, the service answered; + // failing the check just because we're being throttled is wrong). + const expectedCodes = config.expectedStatusCodes || [200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429]; if (!expectedCodes.includes(statusCode)) { return false; }