diff --git a/dashcaddy-api/__tests__/update-manager.test.js b/dashcaddy-api/__tests__/update-manager.test.js index fba81cb..5331dec 100644 --- a/dashcaddy-api/__tests__/update-manager.test.js +++ b/dashcaddy-api/__tests__/update-manager.test.js @@ -125,6 +125,239 @@ describe('UpdateManager — Docker image update lifecycle', () => { }); }); + // ─── DC-078: registry digest probe reliability hardening ────────────────── + // Verifies that getLatestImageDigest / getDockerHubDigest / getGhcrDigest / + // fetchWithReliability all apply the IPv4-only + timeout + transient-retry + // policy. Without these guards, the per-hour checkForUpdates() loop on DNS2 + // surfaces AggregateError [ETIMEDOUT] in error.log because the container's + // /etc/resolv.conf returns AAAA records from Technitium whose IPv6 path to + // public registries (Docker Hub, ghcr.io) is intermittently unreachable. + describe('DC-078 registry reliability', () => { + // Use real timers — fetchWithReliability's retry uses setTimeout for + // backoff, which jest's fake timers would block indefinitely. + beforeEach(() => { + jest.useRealTimers(); + }); + afterEach(() => { + jest.useFakeTimers({ doNotFake: ['setImmediate', 'queueMicrotask', 'nextTick'] }); + }); + + it('_httpsRequestOnce sets family: 4 and timeout on the request options', async () => { + let capturedOptions = null; + const req = { + on: jest.fn(), + end: jest.fn(), + destroy: jest.fn(), + }; + https.request.mockImplementation((options, cb) => { + capturedOptions = options; + // Return a 200 immediately so the promise resolves cleanly. + const res = { + statusCode: 200, + headers: {}, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + return req; + }); + + await updateManager._httpsRequestOnce({ + hostname: 'registry-1.docker.io', + path: '/v2/library/nginx/manifests/latest', + headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }, + maxBodyBytes: 65536, + }); + expect(capturedOptions).not.toBeNull(); + expect(capturedOptions.family).toBe(4); + expect(capturedOptions.timeout).toBeGreaterThan(0); + expect(capturedOptions.method).toBe('GET'); + }); + + it('fetchWithReliability retries on transient ETIMEDOUT and eventually succeeds', async () => { + let attempts = 0; + https.request.mockImplementation((options, cb) => { + attempts += 1; + if (attempts === 1) { + // First attempt: emit ETIMEDOUT via the request 'error' event + const reqErr = new Error('request timeout'); + reqErr.code = 'ETIMEDOUT'; + const req = { + on: jest.fn((event, handler) => { + if (event === 'error') setImmediate(() => handler(reqErr)); + }), + end: jest.fn(), + destroy: jest.fn(), + }; + return req; + } + // Second attempt: 200 OK with a digest header + const res = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:abc123def456' }, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + return { on: jest.fn(), end: jest.fn(), destroy: jest.fn() }; + }); + + const result = await updateManager.fetchWithReliability({ + hostname: 'registry-1.docker.io', + path: '/v2/library/nginx/manifests/latest', + }); + expect(attempts).toBe(2); + expect(result.statusCode).toBe(200); + expect(result.headers['docker-content-digest']).toBe('sha256:abc123def456'); + }); + + it('fetchWithReliability does NOT retry on non-transient HTTP errors', async () => { + let attempts = 0; + https.request.mockImplementation((options, cb) => { + attempts += 1; + const res = { + statusCode: 500, + headers: {}, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + return { on: jest.fn(), end: jest.fn(), destroy: jest.fn() }; + }); + const result = await updateManager.fetchWithReliability({ + hostname: 'registry-1.docker.io', + path: '/v2/library/nginx/manifests/latest', + }); + expect(attempts).toBe(1); + expect(result.statusCode).toBe(500); + }); + + it('fetchWithReliability retries up to REGISTRY_MAX_RETRIES then throws', async () => { + let attempts = 0; + https.request.mockImplementation(() => { + attempts += 1; + const reqErr = new Error('connect ETIMEDOUT'); + reqErr.code = 'ETIMEDOUT'; + const req = { + on: jest.fn((event, handler) => { + if (event === 'error') setImmediate(() => handler(reqErr)); + }), + end: jest.fn(), + destroy: jest.fn(), + }; + return req; + }); + await expect(updateManager.fetchWithReliability({ + hostname: 'registry-1.docker.io', + path: '/v2/library/nginx/manifests/latest', + })).rejects.toMatchObject({ code: 'ETIMEDOUT' }); + // 1 initial attempt + REGISTRY_MAX_RETRIES retries + expect(attempts).toBe(1 + 1); + }); + + it('getDockerHubDigest returns digest on 200', async () => { + https.request.mockImplementation((options, cb) => { + const res = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:hubdigest9999' }, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + return { on: jest.fn(), end: jest.fn(), destroy: jest.fn() }; + }); + const digest = await updateManager.getDockerHubDigest('nginx', 'latest'); + expect(digest).toBe('sha256:hubdigest9999'); + }); + + it('getDockerHubDigest acquires bearer token on 401 then returns digest', async () => { + let calls = 0; + https.request.mockImplementation((options, cb) => { + calls += 1; + if (calls === 1) { + // First call to registry-1.docker.io returns 401 with WWW-Authenticate + const res = { + statusCode: 401, + headers: { + 'www-authenticate': 'Bearer realm="https://auth.example.com/token",service="registry.docker.io",scope="repository:library/nginx:pull"', + }, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + } else if (calls === 2) { + // Second call: auth.example.com returns the token JSON + const res = { + statusCode: 200, + headers: {}, + on: jest.fn((event, handler) => { + if (event === 'data') handler(Buffer.from(JSON.stringify({ token: 'jwt-token-xyz' }))); + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + } else { + // Third call: registry-1.docker.io with Bearer header returns the digest + expect(options.headers['Authorization']).toBe('Bearer jwt-token-xyz'); + const res = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:autheddigest7777' }, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + } + return { on: jest.fn(), end: jest.fn(), destroy: jest.fn() }; + }); + const digest = await updateManager.getDockerHubDigest('nginx', 'latest'); + expect(digest).toBe('sha256:autheddigest7777'); + expect(calls).toBe(3); + }); + + it('getGhcrDigest returns digest on 200', async () => { + https.request.mockImplementation((options, cb) => { + expect(options.hostname).toBe('ghcr.io'); + const res = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:ghcrdigest1234' }, + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }), + }; + setImmediate(() => cb(res)); + return { on: jest.fn(), end: jest.fn(), destroy: jest.fn() }; + }); + const digest = await updateManager.getGhcrDigest('ghcr.io/some/repo', 'latest'); + expect(digest).toBe('sha256:ghcrdigest1234'); + }); + + it('getLatestImageDigest returns null on transient errors after retries (registry unavailable)', async () => { + // Simulate a totally-down registry: every attempt fails with ETIMEDOUT. + // After REGISTRY_MAX_RETRIES the error propagates to getLatestImageDigest's + // catch arm, which logs and returns null (matches old behavior). + https.request.mockImplementation(() => { + const reqErr = new Error('connect ETIMEDOUT'); + reqErr.code = 'ETIMEDOUT'; + const req = { + on: jest.fn((event, handler) => { + if (event === 'error') setImmediate(() => handler(reqErr)); + }), + end: jest.fn(), + destroy: jest.fn(), + }; + return req; + }); + const digest = await updateManager.getLatestImageDigest('nginx:latest'); + expect(digest).toBeNull(); + }); + }); + describe('parseAuthHeader', () => { it('parses Docker Hub Bearer auth header', () => { const header = 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/nginx:pull"'; @@ -481,7 +714,9 @@ describe('UpdateManager — Docker image update lifecycle', () => { setImmediate(() => cb({ statusCode: 200, headers: { 'docker-content-digest': 'sha256:fromregistry' }, - on: jest.fn() + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }) })); return { on: jest.fn(), end: jest.fn() }; }); @@ -495,7 +730,9 @@ describe('UpdateManager — Docker image update lifecycle', () => { setImmediate(() => cb({ statusCode: 401, headers: {}, - on: jest.fn() + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }) })); return { on: jest.fn(), end: jest.fn() }; }); @@ -504,6 +741,9 @@ describe('UpdateManager — Docker image update lifecycle', () => { }); it('rejects on https request error', async () => { + // ECONNREFUSED is in REGISTRY_TRANSIENT_ERROR_CODES, so this would retry. + // Use a non-transient code (or no code) for the test to propagate. + jest.useRealTimers(); https.request.mockImplementation(() => { const req = { on: jest.fn(), end: jest.fn() }; // Trigger error event asynchronously @@ -516,6 +756,7 @@ describe('UpdateManager — Docker image update lifecycle', () => { await expect(updateManager.getDockerHubDigest('nginx', 'latest')) .rejects.toThrow('connection refused'); + jest.useFakeTimers({ doNotFake: ['setImmediate', 'queueMicrotask', 'nextTick'] }); }); it('normalizes library/ prefix for official images', async () => { @@ -525,7 +766,9 @@ describe('UpdateManager — Docker image update lifecycle', () => { setImmediate(() => cb({ statusCode: 200, headers: { 'docker-content-digest': 'sha256:digest' }, - on: jest.fn() + on: jest.fn((event, handler) => { + if (event === 'end') setImmediate(handler); + }) })); return { on: jest.fn(), end: jest.fn() }; }); diff --git a/dashcaddy-api/src/managers/update-manager.js b/dashcaddy-api/src/managers/update-manager.js index ae11d38..b1cdaad 100644 --- a/dashcaddy-api/src/managers/update-manager.js +++ b/dashcaddy-api/src/managers/update-manager.js @@ -18,6 +18,30 @@ const UPDATE_CONFIG_FILE = process.env.UPDATE_CONFIG_FILE || path.join(platformP const UPDATE_HISTORY_FILE = process.env.UPDATE_HISTORY_FILE || path.join(platformPaths.dataDir, 'update-history.json'); const CHECK_INTERVAL = parseInt(process.env.UPDATE_CHECK_INTERVAL || '3600000', 10); // 1 hour +// DC-078: registry probe reliability knobs. The container's /etc/resolv.conf points +// at Technitium (100.121.150.22) which sometimes returns a mix of A and AAAA +// records even when the host's IPv6 path to public registries (Docker Hub, +// ghcr.io) is broken or slow. Without `family: 4` Node defaults to dual-stack, +// every `https.request` to a registry races dual-stack DNS and stalls 30+ seconds +// per ENETUNREACH on the unreachable family. Without an explicit request timeout +// the entire `checkForUpdates()` loop (5+ containers) blocks for minutes per +// tick — visible in error.log as AggregateError [ETIMEDOUT] with a stack like +// `at internalConnectMultiple (node:net:1114:18)`. +// +// TUNABLES — keep conservative; the digest check is a background poll, not +// user-facing. Worst-case latency per query: +// 1st attempt: REGISTRY_REQUEST_TIMEOUT_MS (10s) +// 1st retry : REGISTRY_RETRY_BACKOFF_MS + REGISTRY_REQUEST_TIMEOUT_MS (10.5s) +// ───────────────────────────────────────────────────────────────────── +// per-container ceiling: 20.5s (REGISTRY_MAX_RETRIES=1) +const REGISTRY_REQUEST_TIMEOUT_MS = 10000; // hard per-request socket timeout +const REGISTRY_MAX_RETRIES = 1; // extra attempts after first failure +const REGISTRY_RETRY_BACKOFF_MS = 500; // delay before retry (transient blips) +const REGISTRY_TRANSIENT_ERROR_CODES = new Set([ + 'ETIMEDOUT', 'ENOTFOUND', 'ENETUNREACH', 'ECONNRESET', 'EAI_AGAIN', + 'EPIPE', 'ECONNREFUSED', 'EHOSTUNREACH', +]); + class UpdateManager extends EventEmitter { constructor() { super(); @@ -181,87 +205,208 @@ class UpdateManager extends EventEmitter { * Get image digest from GitHub Container Registry (ghcr.io) * Public images are tokenless via the registry-1.docker.io-style bearer flow, * but using ghcr.io's own auth endpoint. + * + * DC-078: hardened — `family: 4` to avoid the dual-stack DNS race when the + * host's IPv6 path is unreachable (was producing AggregateError [ETIMEDOUT] in + * error.log every check cycle). Hard request timeout caps each attempt. */ async getGhcrDigest(repository, tag) { // ghcr.io uses the same OCI distribution spec as Docker Hub const imageRepo = repository.replace(/^ghcr\.io\//, ''); + const res = await this.fetchWithReliability({ + hostname: 'ghcr.io', + path: `/v2/${imageRepo}/manifests/${tag}`, + headers: { + 'Accept': 'application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json' + }, + }); + return res.headers['docker-content-digest'] || null; + } + + /** + * Get image digest from Docker Hub + * + * DC-078: hardened — see getGhcrDigest comment. Resolves a 401 → token via + * `fetchAuthToken`, which itself is wrapped in the same retry + IPv4-only + + * timeout policy via `fetchWithReliability`. + */ + async getDockerHubDigest(repository, tag) { + // Normalize repository name + const repo = repository.includes('/') ? repository : `library/${repository}`; + const firstAttempt = await this.fetchWithReliability({ + hostname: 'registry-1.docker.io', + path: `/v2/${repo}/manifests/${tag}`, + headers: { + 'Accept': 'application/vnd.docker.distribution.manifest.v2+json' + }, + }); + if (firstAttempt.statusCode !== 401) { + if (firstAttempt.statusCode < 200 || firstAttempt.statusCode >= 300) { + throw new Error(`Docker Hub registry returned HTTP ${firstAttempt.statusCode}`); + } + return firstAttempt.headers['docker-content-digest'] || null; + } + // 401 → acquire a Bearer token via the WWW-Authenticate realm, then retry once. + const authHeader = firstAttempt.headers['www-authenticate']; + const authUrl = this.parseAuthHeader(authHeader); + if (!authUrl) { + throw new Error('Authentication required but no auth URL found'); + } + const token = await this.fetchAuthToken(authUrl); + const authed = await this.fetchWithReliability({ + hostname: 'registry-1.docker.io', + path: `/v2/${repo}/manifests/${tag}`, + headers: { + 'Accept': 'application/vnd.docker.distribution.manifest.v2+json', + 'Authorization': `Bearer ${token}`, + }, + }); + if (authed.statusCode < 200 || authed.statusCode >= 300) { + throw new Error(`Docker Hub registry returned HTTP ${authed.statusCode} after auth`); + } + return authed.headers['docker-content-digest'] || null; + } + + /** + * Single hardened HTTPS probe — DC-078. + * + * Reliability properties: + * 1. `family: 4` — IPv4-only DNS lookup. Avoids dual-stack races where a + * single unreachable IPv6 destination consumes the default 30-second + * connect timeout before the IPv4 fallback succeeds (manifested in + * error.log as AggregateError [ETIMEDOUT] with `at internalConnectMultiple`). + * 2. Hard per-request timeout (REGISTRY_REQUEST_TIMEOUT_MS) — caps total + * latency for any single probe attempt. + * 3. Retry on transient network errors (REGISTRY_TRANSIENT_ERROR_CODES) + * with REGISTRY_RETRY_BACKOFF_MS delay between attempts. Does NOT + * retry on HTTP 4xx/5xx — those are real responses we should surface. + * + * Returns {statusCode, headers, body} so callers can read whichever response + * header or body bytes they need. For digest probes the body is drained and + * discarded; for auth-token fetches the JSON body is parsed. + * + * @param {object} opts + * @param {string} opts.hostname + * @param {string} opts.path + * @param {object} [opts.headers] + * @param {number} [opts.maxBodyBytes=65536] — protect against runaway bodies + */ + async fetchWithReliability(opts) { + const maxBodyBytes = opts.maxBodyBytes || 65536; + let attempt = 0; + while (attempt <= REGISTRY_MAX_RETRIES) { + try { + const result = await this._httpsRequestOnce({ + hostname: opts.hostname, + path: opts.path, + headers: opts.headers || {}, + maxBodyBytes, + }); + return result; + } catch (error) { + // Drain retryable transient errors; non-transient (HTTP status) errors + // and code-less errors are surfaced directly to the caller. + if (!REGISTRY_TRANSIENT_ERROR_CODES.has(error && error.code)) { + throw error; + } + if (attempt >= REGISTRY_MAX_RETRIES) { + throw error; + } + attempt += 1; + // Brief backoff before retry to let transient blips settle. + await new Promise((resolve) => setTimeout(resolve, REGISTRY_RETRY_BACKOFF_MS)); + } + } + // Defensive — should not reach here because the loop either throws or returns. + throw new Error('fetchWithReliability exhausted retries'); + } + + /** + * One-shot HTTPS request helper for fetchWithReliability — DC-078. + * Returns {statusCode, headers, body} on 2xx and most non-2xx responses + * (the caller decides what to do with non-2xx). Throws on transient + * network errors so the retry policy catches them. + */ + _httpsRequestOnce({ hostname, path: urlPath, headers, maxBodyBytes }) { return new Promise((resolve, reject) => { const options = { - hostname: 'ghcr.io', - path: `/v2/${imageRepo}/manifests/${tag}`, + hostname, + path: urlPath, method: 'GET', - headers: { - 'Accept': 'application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json' - } + family: 4, // DC-078: IPv4-only — see top-of-file comment + headers, + timeout: REGISTRY_REQUEST_TIMEOUT_MS, // DC-078: hard per-request cap }; - const req = https.request(options, (res) => { - if (res.statusCode === 401) { - const authHeader = res.headers['www-authenticate']; - const authUrl = this.parseAuthHeader(authHeader); - if (authUrl) { - // ghcr.io auth endpoint accepts scope=repository:owner/name:pull - this.authenticateAndGetDigest(authUrl, options).then(resolve).catch(reject); - } else { - reject(new Error('Authentication required but no auth URL found')); + let body = ''; + let size = 0; + let aborted = false; + res.on('data', (chunk) => { + if (aborted) return; + size += chunk.length; + if (size > maxBodyBytes) { + aborted = true; + res.destroy(); + const err = new Error(`response from ${hostname}${urlPath} exceeded ${maxBodyBytes} bytes`); + err.code = 'ERR_RESPONSE_TOO_LARGE'; + reject(err); + return; } - return; - } - - if (res.statusCode !== 200) { - // Drain body to avoid socket leak - res.resume(); - reject(new Error(`ghcr.io returned HTTP ${res.statusCode}`)); - return; - } - - const digest = res.headers['docker-content-digest']; - resolve(digest || null); + body += chunk; + }); + res.on('end', () => { + if (aborted) return; + resolve({ + statusCode: res.statusCode, + headers: res.headers, + body, + }); + }); + }); + // Node 22 emits 'timeout' on the request, not the socket, when socket.setTimeout + // is hit — make it an explicit error so fetchWithReliability's retry policy catches it. + req.on('timeout', () => { + req.destroy(new Error('request timeout')); + const err = new Error(`registry request to ${hostname}${urlPath} timed out after ${REGISTRY_REQUEST_TIMEOUT_MS}ms`); + err.code = 'ETIMEDOUT'; + reject(err); + }); + req.on('error', (err) => { + // Tag errors missing .code so the retry policy recognizes transient ones. + if (!err.code && /timeout/i.test(err.message)) err.code = 'ETIMEDOUT'; + reject(err); }); - - req.on('error', reject); req.end(); }); } /** - * Get image digest from Docker Hub + * Fetch an auth token from a registry's WWW-Authenticate realm URL — DC-078. + * Uses fetchWithReliability for IPv4-only + timeout + retry. Parses the + * JSON body and returns the `token` or `access_token` field. */ - async getDockerHubDigest(repository, tag) { - return new Promise((resolve, reject) => { - // Normalize repository name - const repo = repository.includes('/') ? repository : `library/${repository}`; - - const options = { - hostname: 'registry-1.docker.io', - path: `/v2/${repo}/manifests/${tag}`, - method: 'GET', - headers: { - 'Accept': 'application/vnd.docker.distribution.manifest.v2+json' - } - }; - - const req = https.request(options, (res) => { - if (res.statusCode === 401) { - // Need to authenticate - const authHeader = res.headers['www-authenticate']; - const authUrl = this.parseAuthHeader(authHeader); - - if (authUrl) { - this.authenticateAndGetDigest(authUrl, options).then(resolve).catch(reject); - } else { - reject(new Error('Authentication required but no auth URL found')); - } - return; - } - - const digest = res.headers['docker-content-digest']; - resolve(digest || null); - }); - - req.on('error', reject); - req.end(); + async fetchAuthToken(authUrl) { + const url = new URL(authUrl); + const result = await this.fetchWithReliability({ + hostname: url.hostname, + path: url.pathname + url.search, + maxBodyBytes: 16384, // auth tokens are <2 KB; cap to a small bound }); + if (result.statusCode !== 200) { + throw new Error(`auth token endpoint ${authUrl} returned HTTP ${result.statusCode}`); + } + let auth; + try { + auth = JSON.parse(result.body); + } catch (parseErr) { + // Surface a clean error — otherwise a malformed token response throws + // SyntaxError with the raw body snippet, which is hard to diagnose + // against the offending realm URL in a log line. + throw new Error(`auth token response from ${authUrl} was not valid JSON: ${parseErr.message}`); + } + const token = auth.token || auth.access_token; + if (!token) throw new Error(`No token in auth response from ${authUrl}`); + return token; } /** @@ -283,48 +428,6 @@ class UpdateManager extends EventEmitter { return url.toString(); } - /** - * Authenticate and get digest - */ - async authenticateAndGetDigest(authUrl, originalOptions) { - return new Promise((resolve, reject) => { - https.get(authUrl, (res) => { - let data = ''; - res.on('data', chunk => data += chunk); - res.on('end', () => { - try { - const auth = JSON.parse(data); - const token = auth.token || auth.access_token; - - if (!token) { - reject(new Error('No token in auth response')); - return; - } - - // Retry original request with token - const options = { - ...originalOptions, - headers: { - ...originalOptions.headers, - 'Authorization': `Bearer ${token}` - } - }; - - const req = https.request(options, (res) => { - const digest = res.headers['docker-content-digest']; - resolve(digest || null); - }); - - req.on('error', reject); - req.end(); - } catch (error) { - reject(error); - } - }); - }).on('error', reject); - }); - } - /** * Extract tag from image name */