dashcaddy/dashcaddy-api/routes/health.js

const express = require('express');
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const { TIMEOUTS } = require('../src/utilities/constants');
const { exists } = require('../src/utilities/fs-helpers');
const { paginate, parsePaginationParams } = require('../src/utilities/pagination');
const platformPaths = require('../platform-paths');
const { resolveServiceUrl } = require('../src/utilities/url-resolver');
const { success, error: errorResponse, errorResponse: sendError, ok, notFound } = require('../src/utils/responses');
const { ValidationError } = require('../src/utilities/errors');

/**
 * Health routes factory
 * @param {Object} deps - Explicit dependencies
 * @param {Function} deps.fetchT - Fetch wrapper with timeout
 * @param {string} deps.SERVICES_FILE - Path to services.json
 * @param {Object} deps.servicesStateManager - State manager for services.json
 * @param {Object} deps.siteConfig - Site configuration
 * @param {Function} deps.buildServiceUrl - URL builder function
 * @param {Function} deps.asyncHandler - Async route handler wrapper
 * @param {Function} deps.logError - Error logging function
 * @param {Object} deps.healthChecker - Health check manager instance
 * @returns {express.Router}
 */
module.exports = function({
  fetchT,
  SERVICES_FILE,
  servicesStateManager,
  siteConfig,
  buildServiceUrl,
  asyncHandler,
  logError,
  healthChecker
}) {
  const router = express.Router();

  // In-memory cache for health results (local to this router)
  let serviceHealthCache = {};
  let lastHealthCheck = null;

  /**
   * Check a URL directly via fetch. Returns health object or null on failure.
   */
  async function checkDirect(url) {
    // Try HEAD first
    try {
      const controller = new AbortController();
      const timeout = setTimeout(() => controller.abort(), TIMEOUTS.HTTP_DEFAULT);
      const response = await fetchT(url, { method: 'HEAD', signal: controller.signal, redirect: 'follow' });
      clearTimeout(timeout);
      return {
        status: response.ok || response.status < 500 ? 'healthy' : 'unhealthy',
        statusCode: response.status,
        url,
        checkedAt: new Date().toISOString()
      };
    } catch { /* ignore */ }

    // Fallback to GET
    try {
      const controller = new AbortController();
      const timeout = setTimeout(() => controller.abort(), TIMEOUTS.HTTP_DEFAULT);
      const response = await fetchT(url, { method: 'GET', signal: controller.signal, redirect: 'follow' });
      clearTimeout(timeout);
      return {
        status: response.ok || response.status < 500 ? 'healthy' : 'unhealthy',
        statusCode: response.status,
        url,
        checkedAt: new Date().toISOString()
      };
    } catch (e) {
      return null; // Direct check completely failed
    }
  }

  /**
   * Check a URL through a Pylon relay. Returns health object or null on failure.
   */
  async function checkViaPylon(pylonConfig, url) {
    if (!pylonConfig?.url) return null;
    try {
      const probeUrl = `${pylonConfig.url}/probe?url=${encodeURIComponent(url)}`;
      const headers = {};
      if (pylonConfig.key) headers['x-pylon-key'] = pylonConfig.key;
      const controller = new AbortController();
      const timeout = setTimeout(() => controller.abort(), 12000);
      const response = await fetchT(probeUrl, { method: 'GET', signal: controller.signal, headers });
      clearTimeout(timeout);
      if (!response.ok) return null;
      const data = await response.json();
      return {
        status: data.status || 'unhealthy',
        statusCode: data.statusCode,
        responseTime: data.responseTime,
        reason: data.reason,
        url,
        via: 'pylon',
        checkedAt: data.checkedAt || new Date().toISOString()
      };
    } catch (_) {
      return null;
    }
  }

  // ===== HEALTH / SERVICES =====

  // Check health of all services (performs live checks)
  router.get('/health/services', asyncHandler(async (req, res) => {
    if (!await exists(SERVICES_FILE)) {
      return success(res, { health: {} });
    }

    const servicesData = await servicesStateManager.read();
    const services = Array.isArray(servicesData) ? servicesData : servicesData.services || [];
    const health = {};
    const pylonConfig = siteConfig?.pylon;

    // Check each service
    await Promise.all(services.map(async (service) => {
      const serviceId = service.id || service.name?.toLowerCase();
      if (!serviceId) return;

      try {
        const url = resolveServiceUrl(serviceId, service, siteConfig, buildServiceUrl);
        if (!url) {
          health[serviceId] = { status: 'unknown', reason: 'No URL configured' };
          return;
        }

        // 1. Try direct check
        let result = await checkDirect(url);
        if (result) {
          health[serviceId] = result;
          return;
        }

        // 2. Direct failed — try through pylon relay
        if (pylonConfig) {
          result = await checkViaPylon(pylonConfig, url);
          if (result) {
            health[serviceId] = result;
            return;
          }
        }

        // 3. Both failed
        health[serviceId] = {
          status: 'unhealthy',
          reason: pylonConfig ? 'Unreachable (direct + pylon)' : 'fetch failed',
          url,
          checkedAt: new Date().toISOString()
        };
      } catch (e) {
        health[serviceId] = {
          status: 'error',
          reason: e.message,
          checkedAt: new Date().toISOString()
        };
      }
    }));

    // Cache results
    serviceHealthCache = health;
    lastHealthCheck = new Date().toISOString();

    const paginationParams = parsePaginationParams(req.query);
    const healthEntries = Object.entries(health);
    const result = paginate(healthEntries, paginationParams);
    const paginatedHealth = Object.fromEntries(result.data);
    success(res, {
      health: paginatedHealth,
      checkedAt: lastHealthCheck,
      ...(result.pagination && { pagination: result.pagination })
    });
  }, 'health-services'));

  // Get cached health status (fast, no re-check)
  router.get('/health/cached', asyncHandler(async (req, res) => {
    success(res, {
      health: serviceHealthCache,
      lastCheck: lastHealthCheck,
      cacheAge: lastHealthCheck ? Date.now() - new Date(lastHealthCheck).getTime() : null
    });
  }, 'health-cached'));

  // Check health of single service
  router.get('/health/service/:id', asyncHandler(async (req, res) => {
    const serviceId = req.params.id;

    // Load service config
    if (!await exists(SERVICES_FILE)) {
      const { NotFoundError } = require('../src/utilities/errors');
      throw new NotFoundError('Services file');
    }

    const servicesData = await servicesStateManager.read();
    const services = Array.isArray(servicesData) ? servicesData : servicesData.services || [];
    const service = services.find(s => (s.id || s.name?.toLowerCase()) === serviceId);

    if (!service) {
      const { NotFoundError } = require('../src/utilities/errors');
      throw new NotFoundError('Service');
    }

    // Determine URL
    const url = resolveServiceUrl(serviceId, service, siteConfig, buildServiceUrl);
    const pylonConfig = siteConfig?.pylon;

    // Try direct, then pylon relay
    let result = await checkDirect(url);
    if (!result && pylonConfig) {
      result = await checkViaPylon(pylonConfig, url);
    }
    if (!result) {
      result = {
        status: 'unhealthy',
        reason: pylonConfig ? 'Unreachable (direct + pylon)' : 'fetch failed',
        url,
        checkedAt: new Date().toISOString()
      };
    }

    success(res, { serviceId, health: result });
  }, 'health-service'));

  // ===== HEALTH / PROBE (Pylon-compatible) =====

  // Probe endpoint — lets this DashCaddy act as a pylon for other instances
  router.get('/health/probe', asyncHandler(async (req, res) => {
    const targetUrl = req.query.url;
    if (!targetUrl) {
      throw new ValidationError('Missing ?url= parameter');
    }
    const result = await checkDirect(targetUrl);
    res.json(result || {
      status: 'unhealthy',
      reason: 'fetch failed',
      url: targetUrl,
      checkedAt: new Date().toISOString()
    });
  }, 'health-probe'));

  // Pylon status — check if the configured pylon is reachable
  router.get('/health/pylon', asyncHandler(async (req, res) => {
    const pylonConfig = siteConfig?.pylon;
    if (!pylonConfig?.url) {
      return success(res, { configured: false });
    }
    try {
      const headers = {};
      if (pylonConfig.key) headers['x-pylon-key'] = pylonConfig.key;
      const response = await fetchT(`${pylonConfig.url}/health`, {
        method: 'GET',
        headers
      }, 5000);
      const data = await response.json();
      success(res, { configured: true, reachable: true, pylon: data });
    } catch (e) {
      success(res, { configured: true, reachable: false, error: e.message });
    }
  }, 'health-pylon'));

  // ===== HEALTH / CA =====

  // Get CA certificate health status
  router.get('/health/ca', asyncHandler(async (req, res) => {
    // Try deployed location first, then Caddy PKI location
    const deployedCertPath = path.join(platformPaths.caCertDir, 'root.crt');
    const pkiCertPath = platformPaths.pkiRootCert;
    const rootCertPath = await exists(deployedCertPath) ? deployedCertPath : pkiCertPath;

    try {
      // Check if certificate exists
      if (!await exists(rootCertPath)) {
        return sendError(res, 404, 'Root CA certificate not found', { caStatus: 'error', daysUntilExpiration: null });
      }

      const dates = execSync(`openssl x509 -in "${rootCertPath}" -noout -dates`).toString();
      const notAfter = dates.match(/notAfter=(.*)/)[1].trim();
      const expirationDate = new Date(notAfter);
      const daysUntilExpiration = Math.floor((expirationDate - new Date()) / (1000 * 60 * 60 * 24));

      // Alert thresholds
      let caStatus = 'healthy';
      let message = `CA certificate valid for ${daysUntilExpiration} days`;

      if (daysUntilExpiration < 0) {
        caStatus = 'critical';
        message = `CA certificate EXPIRED ${Math.abs(daysUntilExpiration)} days ago!`;
      } else if (daysUntilExpiration < 7) {
        caStatus = 'critical';
        message = `CA certificate expires in ${daysUntilExpiration} days!`;
      } else if (daysUntilExpiration < 30) {
        caStatus = 'critical';
        message = `CA certificate expires in ${daysUntilExpiration} days!`;
      } else if (daysUntilExpiration < 90) {
        caStatus = 'warning';
        message = `CA certificate expires in ${daysUntilExpiration} days`;
      }

      ok(res, {
        caStatus,
        message,
        daysUntilExpiration,
        expiresAt: notAfter
      });
    } catch (error) {
      await logError('GET /api/health/ca', error);
      sendError(res, 500, error.message, { caStatus: 'error', daysUntilExpiration: null });
    }
  }, 'health-ca'));

  // ===== HEALTH CHECK (health-checker module) =====

  // Get current status for all services
  // Returns {status: {...per-service}} plus a {summary} block for the System Overview widget
  // — see skill references/totp-and-system-overview-pitfalls.md §3
  router.get('/health-checks/status', asyncHandler(async (req, res) => {
    const status = healthChecker.getCurrentStatus();
    const entries = Object.values(status || {});
    // Treat 'up'/'healthy' as healthy, everything else as unhealthy.
    // Health check status values come from healthChecker — typically 'up'/'down' but
    // also 'healthy'/'unhealthy' or 'online'/'offline' depending on the source. Be
    // permissive on the healthy side so a service in any positive state counts.
    const healthy = entries.filter(s => {
      const st = (s && (s.status || s.state)) || '';
      return st === 'up' || st === 'healthy' || st === 'online';
    }).length;
    const unhealthy = entries.filter(s => {
      const st = (s && (s.status || s.state)) || '';
      return st === 'down' || st === 'unhealthy' || st === 'offline' || st === 'error';
    }).length;
    const unknown = entries.length - healthy - unhealthy;
    const summary = { healthy, unhealthy, unknown, total: entries.length };
    success(res, { status, summary });
  }, 'health-check-status'));

  // Get service statistics
  router.get('/health-checks/:serviceId/stats', asyncHandler(async (req, res) => {
    const hours = parseInt(req.query.hours) || 24;
    const stats = healthChecker.getServiceStats(req.params.serviceId, hours);
    if (!stats) {
      const { NotFoundError } = require('../src/utilities/errors');
      throw new NotFoundError('Service');
    }
    success(res, { stats });
  }, 'health-check-stats'));

  // Configure health check
  router.post('/health-checks/:serviceId/configure', asyncHandler(async (req, res) => {
    healthChecker.configureService(req.params.serviceId, req.body);
    success(res, { message: 'Health check configured' });
  }, 'health-check-configure'));

  // Remove health check configuration
  router.delete('/health-checks/:serviceId/configure', asyncHandler(async (req, res) => {
    healthChecker.removeService(req.params.serviceId);
    success(res, { message: 'Health check removed' });
  }, 'health-check-remove'));

  // Get open incidents
  router.get('/health-checks/incidents', asyncHandler(async (req, res) => {
    const incidents = healthChecker.getOpenIncidents();
    const paginationParams = parsePaginationParams(req.query);
    const result = paginate(incidents, paginationParams);
    success(res, { incidents: result.data, ...(result.pagination && { pagination: result.pagination }) });
  }, 'health-check-incidents'));

  // Get incident history
  router.get('/health-checks/incidents/history', asyncHandler(async (req, res) => {
    const paginationParams = parsePaginationParams(req.query);
    // When paginating, fetch all history so pagination can slice correctly
    const fetchLimit = paginationParams ? Number.MAX_SAFE_INTEGER : (parseInt(req.query.limit) || 50);
    const history = healthChecker.getIncidentHistory(fetchLimit);
    const result = paginate(history, paginationParams);
    success(res, { history: result.data, ...(result.pagination && { pagination: result.pagination }) });
  }, 'health-check-incidents-history'));

  // ── DC-075: System health endpoint for operators/uptime monitoring ─────────
  // Returns a single "is everything OK" summary suitable for external monitors
  // like UptimeRobot or BetterStack. No auth required (read-only status).
  router.get('/system/health', asyncHandler(async (req, res) => {
    const checks = {};

    // Service health from health checker
    try {
      const status = healthChecker.getCurrentStatus();
      const entries = Object.values(status || {});
      const unhealthy = entries.filter(s => {
        const st = (s && (s.status || s.state)) || '';
        return st === 'down' || st === 'unhealthy' || st === 'offline' || st === 'error';
      }).length;
      const total = entries.length;
      const knownHealthy = entries.filter(s => {
        const st = (s && (s.status || s.state)) || '';
        return st === 'up' || st === 'healthy' || st === 'online';
      }).length;
      checks.services = {
        status: unhealthy === 0 ? 'ok' : (unhealthy < total ? 'degraded' : 'down'),
        healthy: knownHealthy,
        unhealthy,
        unknown: total - knownHealthy - unhealthy,
        total,
      };
    } catch {
      checks.services = { status: 'unknown' };
    }

    // Memory usage
    try {
      const os = require('os');
      const total = os.totalmem ? os.totalmem() : 0;
      const free = os.freemem ? os.freemem() : 0;
      checks.memory = {
        status: free / total > 0.1 ? 'ok' : 'warning',
        usedPercent: parseFloat((((total - free) / total) * 100).toFixed(1)),
        totalMB: Math.round(total / 1048576),
        freeMB: Math.round(free / 1048576),
      };
    } catch {
      checks.memory = { status: 'unknown' };
    }

    // Disk space (data dir)
    try {
      const { execSync } = require('child_process');
      const dfOutput = execSync('df -h --output=pcent,size,avail ' + (platformPaths.dataDir || '/'), { encoding: 'utf8', timeout: 3000 });
      const lines = dfOutput.trim().split('\n');
      if (lines.length >= 2) {
        const parts = lines[1].trim().split(/\s+/);
        const usedPercent = parseInt(parts[0]);
        checks.diskSpace = {
          status: usedPercent < 90 ? 'ok' : (usedPercent < 95 ? 'warning' : 'critical'),
          usedPercent,
          total: parts[1],
          available: parts[2],
        };
      }
    } catch {
      checks.diskSpace = { status: 'unknown' };
    }

    // Uptime
    const uptime = process.uptime();
    checks.uptime = {
      seconds: Math.round(uptime),
      human: `${Math.floor(uptime / 3600)}h ${Math.floor((uptime % 3600) / 60)}m`,
    };

    // Open incidents
    try {
      const incidents = healthChecker.getOpenIncidents();
      checks.incidents = {
        status: incidents.length === 0 ? 'ok' : 'degraded',
        count: incidents.length,
      };
    } catch {
      checks.incidents = { status: 'unknown', count: 0 };
    }

    // Overall status: 'unknown' is treated as degraded (not healthy)
    const statuses = Object.values(checks).map(c => c.status);
    const overall = statuses.includes('down') || statuses.includes('critical') ? 'unhealthy'
      : statuses.some(s => s === 'degraded' || s === 'warning' || s === 'unknown') ? 'degraded'
      : 'healthy';

    res.set('Cache-Control', 'no-store');
    success(res, {
      status: overall,
      timestamp: new Date().toISOString(),
      checks,
    });
  }, 'system-health'));

  return router;
};