const express = require('express'); const fs = require('fs'); const path = require('path'); const { execSync } = require('child_process'); const { TIMEOUTS } = require('../src/utilities/constants'); const { exists } = require('../src/utilities/fs-helpers'); const { paginate, parsePaginationParams } = require('../src/utilities/pagination'); const platformPaths = require('../platform-paths'); const { resolveServiceUrl } = require('../src/utilities/url-resolver'); const { success, error: errorResponse, errorResponse: sendError, ok, notFound } = require('../src/utils/responses'); const { ValidationError } = require('../src/utilities/errors'); /** * Health routes factory * @param {Object} deps - Explicit dependencies * @param {Function} deps.fetchT - Fetch wrapper with timeout * @param {string} deps.SERVICES_FILE - Path to services.json * @param {Object} deps.servicesStateManager - State manager for services.json * @param {Object} deps.siteConfig - Site configuration * @param {Function} deps.buildServiceUrl - URL builder function * @param {Function} deps.asyncHandler - Async route handler wrapper * @param {Function} deps.logError - Error logging function * @param {Object} deps.healthChecker - Health check manager instance * @returns {express.Router} */ module.exports = function({ fetchT, SERVICES_FILE, servicesStateManager, siteConfig, buildServiceUrl, asyncHandler, logError, healthChecker }) { const router = express.Router(); // In-memory cache for health results (local to this router) let serviceHealthCache = {}; let lastHealthCheck = null; /** * Check a URL directly via fetch. Returns health object or null on failure. */ async function checkDirect(url) { // Try HEAD first try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), TIMEOUTS.HTTP_DEFAULT); const response = await fetchT(url, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }); clearTimeout(timeout); return { status: response.ok || response.status < 500 ? 'healthy' : 'unhealthy', statusCode: response.status, url, checkedAt: new Date().toISOString() }; } catch { /* ignore */ } // Fallback to GET try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), TIMEOUTS.HTTP_DEFAULT); const response = await fetchT(url, { method: 'GET', signal: controller.signal, redirect: 'follow' }); clearTimeout(timeout); return { status: response.ok || response.status < 500 ? 'healthy' : 'unhealthy', statusCode: response.status, url, checkedAt: new Date().toISOString() }; } catch (e) { return null; // Direct check completely failed } } /** * Check a URL through a Pylon relay. Returns health object or null on failure. */ async function checkViaPylon(pylonConfig, url) { if (!pylonConfig?.url) return null; try { const probeUrl = `${pylonConfig.url}/probe?url=${encodeURIComponent(url)}`; const headers = {}; if (pylonConfig.key) headers['x-pylon-key'] = pylonConfig.key; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 12000); const response = await fetchT(probeUrl, { method: 'GET', signal: controller.signal, headers }); clearTimeout(timeout); if (!response.ok) return null; const data = await response.json(); return { status: data.status || 'unhealthy', statusCode: data.statusCode, responseTime: data.responseTime, reason: data.reason, url, via: 'pylon', checkedAt: data.checkedAt || new Date().toISOString() }; } catch (_) { return null; } } // ===== HEALTH / SERVICES ===== // Check health of all services (performs live checks) router.get('/health/services', asyncHandler(async (req, res) => { if (!await exists(SERVICES_FILE)) { return success(res, { health: {} }); } const servicesData = await servicesStateManager.read(); const services = Array.isArray(servicesData) ? servicesData : servicesData.services || []; const health = {}; const pylonConfig = siteConfig?.pylon; // Check each service await Promise.all(services.map(async (service) => { const serviceId = service.id || service.name?.toLowerCase(); if (!serviceId) return; try { const url = resolveServiceUrl(serviceId, service, siteConfig, buildServiceUrl); if (!url) { health[serviceId] = { status: 'unknown', reason: 'No URL configured' }; return; } // 1. Try direct check let result = await checkDirect(url); if (result) { health[serviceId] = result; return; } // 2. Direct failed — try through pylon relay if (pylonConfig) { result = await checkViaPylon(pylonConfig, url); if (result) { health[serviceId] = result; return; } } // 3. Both failed health[serviceId] = { status: 'unhealthy', reason: pylonConfig ? 'Unreachable (direct + pylon)' : 'fetch failed', url, checkedAt: new Date().toISOString() }; } catch (e) { health[serviceId] = { status: 'error', reason: e.message, checkedAt: new Date().toISOString() }; } })); // Cache results serviceHealthCache = health; lastHealthCheck = new Date().toISOString(); const paginationParams = parsePaginationParams(req.query); const healthEntries = Object.entries(health); const result = paginate(healthEntries, paginationParams); const paginatedHealth = Object.fromEntries(result.data); success(res, { health: paginatedHealth, checkedAt: lastHealthCheck, ...(result.pagination && { pagination: result.pagination }) }); }, 'health-services')); // Get cached health status (fast, no re-check) router.get('/health/cached', asyncHandler(async (req, res) => { success(res, { health: serviceHealthCache, lastCheck: lastHealthCheck, cacheAge: lastHealthCheck ? Date.now() - new Date(lastHealthCheck).getTime() : null }); }, 'health-cached')); // Check health of single service router.get('/health/service/:id', asyncHandler(async (req, res) => { const serviceId = req.params.id; // Load service config if (!await exists(SERVICES_FILE)) { const { NotFoundError } = require('../src/utilities/errors'); throw new NotFoundError('Services file'); } const servicesData = await servicesStateManager.read(); const services = Array.isArray(servicesData) ? servicesData : servicesData.services || []; const service = services.find(s => (s.id || s.name?.toLowerCase()) === serviceId); if (!service) { const { NotFoundError } = require('../src/utilities/errors'); throw new NotFoundError('Service'); } // Determine URL const url = resolveServiceUrl(serviceId, service, siteConfig, buildServiceUrl); const pylonConfig = siteConfig?.pylon; // Try direct, then pylon relay let result = await checkDirect(url); if (!result && pylonConfig) { result = await checkViaPylon(pylonConfig, url); } if (!result) { result = { status: 'unhealthy', reason: pylonConfig ? 'Unreachable (direct + pylon)' : 'fetch failed', url, checkedAt: new Date().toISOString() }; } success(res, { serviceId, health: result }); }, 'health-service')); // ===== HEALTH / PROBE (Pylon-compatible) ===== // Probe endpoint — lets this DashCaddy act as a pylon for other instances router.get('/health/probe', asyncHandler(async (req, res) => { const targetUrl = req.query.url; if (!targetUrl) { throw new ValidationError('Missing ?url= parameter'); } const result = await checkDirect(targetUrl); res.json(result || { status: 'unhealthy', reason: 'fetch failed', url: targetUrl, checkedAt: new Date().toISOString() }); }, 'health-probe')); // Pylon status — check if the configured pylon is reachable router.get('/health/pylon', asyncHandler(async (req, res) => { const pylonConfig = siteConfig?.pylon; if (!pylonConfig?.url) { return success(res, { configured: false }); } try { const headers = {}; if (pylonConfig.key) headers['x-pylon-key'] = pylonConfig.key; const response = await fetchT(`${pylonConfig.url}/health`, { method: 'GET', headers }, 5000); const data = await response.json(); success(res, { configured: true, reachable: true, pylon: data }); } catch (e) { success(res, { configured: true, reachable: false, error: e.message }); } }, 'health-pylon')); // ===== HEALTH / CA ===== // Get CA certificate health status router.get('/health/ca', asyncHandler(async (req, res) => { // Try deployed location first, then Caddy PKI location const deployedCertPath = path.join(platformPaths.caCertDir, 'root.crt'); const pkiCertPath = platformPaths.pkiRootCert; const rootCertPath = await exists(deployedCertPath) ? deployedCertPath : pkiCertPath; try { // Check if certificate exists if (!await exists(rootCertPath)) { return sendError(res, 404, 'Root CA certificate not found', { caStatus: 'error', daysUntilExpiration: null }); } const dates = execSync(`openssl x509 -in "${rootCertPath}" -noout -dates`).toString(); const notAfter = dates.match(/notAfter=(.*)/)[1].trim(); const expirationDate = new Date(notAfter); const daysUntilExpiration = Math.floor((expirationDate - new Date()) / (1000 * 60 * 60 * 24)); // Alert thresholds let caStatus = 'healthy'; let message = `CA certificate valid for ${daysUntilExpiration} days`; if (daysUntilExpiration < 0) { caStatus = 'critical'; message = `CA certificate EXPIRED ${Math.abs(daysUntilExpiration)} days ago!`; } else if (daysUntilExpiration < 7) { caStatus = 'critical'; message = `CA certificate expires in ${daysUntilExpiration} days!`; } else if (daysUntilExpiration < 30) { caStatus = 'critical'; message = `CA certificate expires in ${daysUntilExpiration} days!`; } else if (daysUntilExpiration < 90) { caStatus = 'warning'; message = `CA certificate expires in ${daysUntilExpiration} days`; } ok(res, { caStatus, message, daysUntilExpiration, expiresAt: notAfter }); } catch (error) { await logError('GET /api/health/ca', error); sendError(res, 500, error.message, { caStatus: 'error', daysUntilExpiration: null }); } }, 'health-ca')); // ===== HEALTH CHECK (health-checker module) ===== // Get current status for all services // Returns {status: {...per-service}} plus a {summary} block for the System Overview widget // — see skill references/totp-and-system-overview-pitfalls.md §3 router.get('/health-checks/status', asyncHandler(async (req, res) => { const status = healthChecker.getCurrentStatus(); const entries = Object.values(status || {}); // Treat 'up'/'healthy' as healthy, everything else as unhealthy. // Health check status values come from healthChecker — typically 'up'/'down' but // also 'healthy'/'unhealthy' or 'online'/'offline' depending on the source. Be // permissive on the healthy side so a service in any positive state counts. const healthy = entries.filter(s => { const st = (s && (s.status || s.state)) || ''; return st === 'up' || st === 'healthy' || st === 'online'; }).length; const unhealthy = entries.filter(s => { const st = (s && (s.status || s.state)) || ''; return st === 'down' || st === 'unhealthy' || st === 'offline' || st === 'error'; }).length; const unknown = entries.length - healthy - unhealthy; const summary = { healthy, unhealthy, unknown, total: entries.length }; success(res, { status, summary }); }, 'health-check-status')); // Get service statistics router.get('/health-checks/:serviceId/stats', asyncHandler(async (req, res) => { const hours = parseInt(req.query.hours) || 24; const stats = healthChecker.getServiceStats(req.params.serviceId, hours); if (!stats) { const { NotFoundError } = require('../src/utilities/errors'); throw new NotFoundError('Service'); } success(res, { stats }); }, 'health-check-stats')); // Configure health check router.post('/health-checks/:serviceId/configure', asyncHandler(async (req, res) => { healthChecker.configureService(req.params.serviceId, req.body); success(res, { message: 'Health check configured' }); }, 'health-check-configure')); // Remove health check configuration router.delete('/health-checks/:serviceId/configure', asyncHandler(async (req, res) => { healthChecker.removeService(req.params.serviceId); success(res, { message: 'Health check removed' }); }, 'health-check-remove')); // Get open incidents router.get('/health-checks/incidents', asyncHandler(async (req, res) => { const incidents = healthChecker.getOpenIncidents(); const paginationParams = parsePaginationParams(req.query); const result = paginate(incidents, paginationParams); success(res, { incidents: result.data, ...(result.pagination && { pagination: result.pagination }) }); }, 'health-check-incidents')); // Get incident history router.get('/health-checks/incidents/history', asyncHandler(async (req, res) => { const paginationParams = parsePaginationParams(req.query); // When paginating, fetch all history so pagination can slice correctly const fetchLimit = paginationParams ? Number.MAX_SAFE_INTEGER : (parseInt(req.query.limit) || 50); const history = healthChecker.getIncidentHistory(fetchLimit); const result = paginate(history, paginationParams); success(res, { history: result.data, ...(result.pagination && { pagination: result.pagination }) }); }, 'health-check-incidents-history')); // ── DC-075: System health endpoint for operators/uptime monitoring ───────── // Returns a single "is everything OK" summary suitable for external monitors // like UptimeRobot or BetterStack. No auth required (read-only status). router.get('/system/health', asyncHandler(async (req, res) => { const checks = {}; // Service health from health checker try { const status = healthChecker.getCurrentStatus(); const entries = Object.values(status || {}); const unhealthy = entries.filter(s => { const st = (s && (s.status || s.state)) || ''; return st === 'down' || st === 'unhealthy' || st === 'offline' || st === 'error'; }).length; const total = entries.length; const knownHealthy = entries.filter(s => { const st = (s && (s.status || s.state)) || ''; return st === 'up' || st === 'healthy' || st === 'online'; }).length; checks.services = { status: unhealthy === 0 ? 'ok' : (unhealthy < total ? 'degraded' : 'down'), healthy: knownHealthy, unhealthy, unknown: total - knownHealthy - unhealthy, total, }; } catch { checks.services = { status: 'unknown' }; } // Memory usage try { const os = require('os'); const total = os.totalmem ? os.totalmem() : 0; const free = os.freemem ? os.freemem() : 0; checks.memory = { status: free / total > 0.1 ? 'ok' : 'warning', usedPercent: parseFloat((((total - free) / total) * 100).toFixed(1)), totalMB: Math.round(total / 1048576), freeMB: Math.round(free / 1048576), }; } catch { checks.memory = { status: 'unknown' }; } // Disk space (data dir) try { const { execSync } = require('child_process'); const dfOutput = execSync('df -h --output=pcent,size,avail ' + (platformPaths.dataDir || '/'), { encoding: 'utf8', timeout: 3000 }); const lines = dfOutput.trim().split('\n'); if (lines.length >= 2) { const parts = lines[1].trim().split(/\s+/); const usedPercent = parseInt(parts[0]); checks.diskSpace = { status: usedPercent < 90 ? 'ok' : (usedPercent < 95 ? 'warning' : 'critical'), usedPercent, total: parts[1], available: parts[2], }; } } catch { checks.diskSpace = { status: 'unknown' }; } // Uptime const uptime = process.uptime(); checks.uptime = { seconds: Math.round(uptime), human: `${Math.floor(uptime / 3600)}h ${Math.floor((uptime % 3600) / 60)}m`, }; // Open incidents try { const incidents = healthChecker.getOpenIncidents(); checks.incidents = { status: incidents.length === 0 ? 'ok' : 'degraded', count: incidents.length, }; } catch { checks.incidents = { status: 'unknown', count: 0 }; } // Overall status: 'unknown' is treated as degraded (not healthy) const statuses = Object.values(checks).map(c => c.status); const overall = statuses.includes('down') || statuses.includes('critical') ? 'unhealthy' : statuses.some(s => s === 'degraded' || s === 'warning' || s === 'unknown') ? 'degraded' : 'healthy'; res.set('Cache-Control', 'no-store'); success(res, { status: overall, timestamp: new Date().toISOString(), checks, }); }, 'system-health')); return router; };