dashcaddy/dashcaddy-api/__tests__/routes/system-health.routes.test.js

/**
 * DC-083: Branch coverage tests for the new /system/health endpoint in routes/health.js.
 *
 * The endpoint at GET /api/system/health aggregates four checks (services, memory,
 * diskSpace, incidents) into an overall status. It has many uncovered branches:
 *  - status === 'ok' / 'degraded' / 'down' in the services check
 *  - status === 'ok' / 'warning' in the memory check
 *  - status === 'ok' / 'warning' / 'critical' in the diskSpace check
 *  - status === 'ok' / 'degraded' in the incidents check
 *  - each check has a try/catch → unknown fallback
 *  - overall status computation (unhealthy / degraded / healthy)
 *
 * Also covers additional uncovered branches in the /health-checks/* endpoints:
 *  - unhealthy filter in /health-checks/status
 *  - incidents open/non-empty
 *  - incidents/history with pagination params
 *  - /health/probe with and without ?url
 *  - /health/services with array vs object services data, error paths
 */
const express = require('express');
const request = require('supertest');

// Minimal asyncHandler that catches errors
function asyncHandler(fn) {
  return (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next);
}

// ---- Mocks (mirrors health.routes.test.js) ----
jest.mock('child_process', () => ({ execSync: jest.fn() }));
jest.mock('../../platform-paths', () => ({
  caCertDir: '/mock/ca',
  pkiRootCert: '/mock/pki/root.crt',
  dataDir: '/mock/data',
}));
jest.mock('../../src/utilities/fs-helpers', () => ({ exists: jest.fn().mockResolvedValue(true) }));
jest.mock('../../src/utilities/url-resolver', () => ({
  resolveServiceUrl: jest.fn((id) => `https://${id}.test`),
}));
jest.mock('../../src/utilities/pagination', () => ({
  paginate: jest.fn((data, params) => ({ data, pagination: params ? { page: 1, limit: 10, total: data.length } : null })),
  parsePaginationParams: jest.fn(() => null),
}));

const { exists } = require('../../src/utilities/fs-helpers');
const { resolveServiceUrl } = require('../../src/utilities/url-resolver');
const { execSync } = require('child_process');
const platformPaths = require('../../platform-paths');

function createApp(depsOverride = {}) {
  const defaultDeps = {
    fetchT: jest.fn().mockResolvedValue({ ok: true, status: 200, json: () => ({}) }),
    SERVICES_FILE: '/tmp/services.json',
    servicesStateManager: {
      read: jest.fn().mockResolvedValue([]),
      write: jest.fn().mockResolvedValue(),
      update: jest.fn().mockResolvedValue([]),
    },
    siteConfig: { tld: 'sami' },
    buildServiceUrl: jest.fn(id => `https://${id}.sami`),
    asyncHandler,
    logError: jest.fn(),
    healthChecker: {
      getCurrentStatus: jest.fn().mockReturnValue({}),
      getServiceStats: jest.fn().mockReturnValue(null),
      configureService: jest.fn(),
      removeService: jest.fn(),
      getOpenIncidents: jest.fn().mockReturnValue([]),
      getIncidentHistory: jest.fn().mockReturnValue([]),
    },
  };
  const deps = { ...defaultDeps, ...depsOverride };
  const healthRoutes = require('../../routes/health');
  const app = express();
  app.use(express.json());
  app.use('/api', healthRoutes(deps));
  app.use((err, req, res, next) => {
    const status = err.statusCode || 500;
    res.status(status).json({ success: false, error: err.message });
  });
  return { app, deps };
}

describe('System health endpoint (DC-083)', () => {
  beforeEach(() => {
    jest.clearAllMocks();
    exists.mockResolvedValue(true);
    execSync.mockReturnValue('notAfter=Dec 22 12:00:00 2034 GMT');
  });

  describe('GET /api/system/health', () => {
    it('returns healthy overall when all checks pass', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: { status: 'up' },
          svc2: { status: 'healthy' },
          svc3: { status: 'online' },
        }),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      // disk: 40% used → ok. df output format: header line + data line.
      // parts[0]='40%', parseInt → 40
      execSync.mockReturnValue('Use%  Size  Avail\n 40%  100G  60G');
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.status).toBe(200);
      expect(res.body.status).toBe('healthy');
      expect(res.body.checks.services.status).toBe('ok');
      expect(res.body.checks.services.healthy).toBe(3);
      expect(res.body.checks.memory.status).toBe('ok');
      expect(res.body.checks.diskSpace.status).toBe('ok');
      expect(res.body.checks.incidents.status).toBe('ok');
    });

    it('returns degraded when some services are unhealthy (mixed)', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: { status: 'up' },
          svc2: { status: 'down' },
        }),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.status).toBe(200);
      expect(res.body.checks.services.status).toBe('degraded');
      expect(res.body.checks.services.unhealthy).toBe(1);
      expect(res.body.checks.services.unknown).toBe(0);
      // Overall degraded because services degraded
      expect(res.body.status).toBe('degraded');
    });

    it('returns down when ALL services are unhealthy', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: { status: 'down' },
          svc2: { status: 'offline' },
        }),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.services.status).toBe('down');
      // Overall unhealthy because services down
      expect(res.body.status).toBe('unhealthy');
    });

    it('counts unknown status values (not up/down/healthy/etc.)', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: { state: 'starting' },     // unknown state value
          svc2: { status: 'paused' },      // unknown status value
          svc3: { },                        // no status/state → unknown
        }),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.services.total).toBe(3);
      expect(res.body.checks.services.healthy).toBe(0);
      expect(res.body.checks.services.unhealthy).toBe(0);
      expect(res.body.checks.services.unknown).toBe(3);
    });

    it('returns degraded when incidents are open', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getOpenIncidents: jest.fn().mockReturnValue([{ id: 'inc1' }, { id: 'inc2' }]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.incidents.status).toBe('degraded');
      expect(res.body.checks.incidents.count).toBe(2);
      expect(res.body.status).toBe('degraded');
    });

    it('returns warning when disk usage between 90-95%', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      execSync.mockReturnValue('Use%  Size  Avail\n 92%  100G  8G');
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.diskSpace.status).toBe('warning');
      expect(res.body.checks.diskSpace.usedPercent).toBe(92);
      expect(res.body.status).toBe('degraded');
    });

    it('returns critical when disk usage >= 95%', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      execSync.mockReturnValue('Use%  Size  Avail\n 97%  100G  3G');
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.diskSpace.status).toBe('critical');
      expect(res.body.status).toBe('unhealthy');
    });

    it('falls back to unknown for services when getCurrentStatus throws', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockImplementation(() => { throw new Error('boom'); }),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.services.status).toBe('unknown');
      // unknown → degraded overall
      expect(res.body.status).toBe('degraded');
    });

    it('falls back to unknown for disk when execSync throws', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      execSync.mockImplementation(() => { throw new Error('df failed'); });
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.diskSpace.status).toBe('unknown');
    });

    it('falls back to unknown for incidents when getOpenIncidents throws', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getOpenIncidents: jest.fn().mockImplementation(() => { throw new Error('inc fail'); }),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.incidents.status).toBe('unknown');
      expect(res.body.checks.incidents.count).toBe(0);
    });

    it('sets Cache-Control: no-store header', async () => {
      const { app } = createApp();
      const res = await request(app).get('/api/system/health');
      expect(res.headers['cache-control']).toBe('no-store');
    });

    it('includes uptime block with seconds and human-readable', async () => {
      const { app } = createApp();
      const res = await request(app).get('/api/system/health');
      expect(res.body.checks.uptime).toHaveProperty('seconds');
      expect(res.body.checks.uptime).toHaveProperty('human');
      expect(typeof res.body.checks.uptime.seconds).toBe('number');
    });

    it('handles empty df output (only header line) — no diskSpace block set to ok', async () => {
      // df returns just one line → lines.length < 2 → diskSpace not assigned in try
      // (stays undefined → overall status considers it). Actually the try block
      // does NOT set diskSpace when lines.length < 2, so diskSpace is undefined
      // and Object.values(checks) excludes it. Verify no crash.
      execSync.mockReturnValue('Use%  Size  Avail');
      const { app } = createApp();
      const res = await request(app).get('/api/system/health');
      expect(res.status).toBe(200);
    });
  });

  // ---- Coverage for health-checks/status unhealthy filter ----
  describe('GET /api/health-checks/status — unhealthy filter coverage', () => {
    it('counts unhealthy services via various status/state tokens', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: { status: 'down' },
          svc2: { state: 'unhealthy' },
          svc3: { status: 'offline' },
          svc4: { status: 'error' },
          svc5: { status: 'up' },
        }),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/health-checks/status');
      expect(res.status).toBe(200);
      expect(res.body.summary.unhealthy).toBe(4);
      expect(res.body.summary.healthy).toBe(1);
      expect(res.body.summary.unknown).toBe(0);
      expect(res.body.summary.total).toBe(5);
    });

    it('handles null/undefined status entries', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({
          svc1: null,
          svc2: {},
          svc3: { status: 'up' },
        }),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getOpenIncidents: jest.fn().mockReturnValue([]),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/health-checks/status');
      expect(res.status).toBe(200);
      // null and {} are not healthy or unhealthy → unknown
      expect(res.body.summary.unknown).toBe(2);
      expect(res.body.summary.healthy).toBe(1);
    });
  });

  // ---- Coverage for /health/probe ----
  describe('GET /api/health/probe', () => {
    it('returns 400 when url query param missing', async () => {
      const { app } = createApp();
      const res = await request(app).get('/api/health/probe');
      expect(res.status).toBe(400);
    });

    it('returns probe result when url provided and fetch succeeds', async () => {
      const fetchT = jest.fn().mockResolvedValue({ ok: true, status: 200, json: () => ({}) });
      const { app } = createApp({ fetchT });
      const res = await request(app).get('/api/health/probe?url=https://example.com');
      expect(res.status).toBe(200);
      expect(res.body.status).toBe('healthy');
      expect(res.body.statusCode).toBe(200);
    });

    it('returns unhealthy when probe fetch fails completely', async () => {
      const fetchT = jest.fn().mockRejectedValue(new Error('timeout'));
      const { app } = createApp({ fetchT });
      const res = await request(app).get('/api/health/probe?url=https://down.example');
      expect(res.status).toBe(200);
      expect(res.body.status).toBe('unhealthy');
      expect(res.body.reason).toBe('fetch failed');
    });

    it('marks status as unhealthy when statusCode >= 500', async () => {
      const fetchT = jest.fn().mockResolvedValue({ ok: false, status: 503 });
      const { app } = createApp({ fetchT });
      const res = await request(app).get('/api/health/probe?url=https://500.example');
      expect(res.body.status).toBe('unhealthy');
      expect(res.body.statusCode).toBe(503);
    });

    it('marks status as healthy when statusCode is 401/403 (auth wall)', async () => {
      const fetchT = jest.fn().mockResolvedValue({ ok: false, status: 401 });
      const { app } = createApp({ fetchT });
      const res = await request(app).get('/api/health/probe?url=https://auth.example');
      expect(res.body.status).toBe('healthy');
      expect(res.body.statusCode).toBe(401);
    });
  });

  // ---- Coverage for /health/services with various service shapes ----
  describe('GET /api/health/services — service shape branches', () => {
    it('handles services as object with .services array', async () => {
      const stateManager = {
        read: jest.fn().mockResolvedValue({ services: [{ id: 'svc1', name: 'S1' }] }),
        write: jest.fn(),
        update: jest.fn(),
      };
      const fetchT = jest.fn().mockResolvedValue({ ok: true, status: 200 });
      const { app } = createApp({ servicesStateManager: stateManager, fetchT });
      const res = await request(app).get('/api/health/services');
      expect(res.status).toBe(200);
      expect(res.body.health).toHaveProperty('svc1');
    });

    it('uses service.name (lowercased) as id when service.id absent', async () => {
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ name: 'MyService' }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const fetchT = jest.fn().mockResolvedValue({ ok: true, status: 200 });
      const { app } = createApp({ servicesStateManager: stateManager, fetchT });
      const res = await request(app).get('/api/health/services');
      expect(res.status).toBe(200);
      expect(res.body.health).toHaveProperty('myservice');
    });

    it('skips services with no id and no name', async () => {
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ port: 8080 }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const { app } = createApp({ servicesStateManager: stateManager });
      const res = await request(app).get('/api/health/services');
      expect(res.status).toBe(200);
      expect(res.body.health).toEqual({});
    });

    it('marks service as unknown when URL resolves to null', async () => {
      resolveServiceUrl.mockReturnValue(null);
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ id: 'novurl', name: 'No URL' }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const { app } = createApp({ servicesStateManager: stateManager });
      const res = await request(app).get('/api/health/services');
      expect(res.body.health.novurl.status).toBe('unknown');
      expect(res.body.health.novurl.reason).toMatch(/No URL/);
      resolveServiceUrl.mockReturnValue('https://fallback.test');
    });

    it('uses pylon relay when direct check fails and pylon configured', async () => {
      // Direct HEAD and GET both throw → falls through to pylon
      const fetchT = jest.fn()
        .mockRejectedValueOnce(new Error('HEAD fail'))   // HEAD
        .mockRejectedValueOnce(new Error('GET fail'))    // GET (fallback in checkDirect)
        .mockResolvedValueOnce({                          // pylon probe
          ok: true, status: 200,
          json: () => ({ status: 'healthy', statusCode: 200, responseTime: 42 }),
        });
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ id: 'svc1', name: 'S1' }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const { app } = createApp({
        servicesStateManager: stateManager,
        fetchT,
        siteConfig: { tld: 'sami', pylon: { url: 'http://pylon.test', key: 'k' } },
      });
      const res = await request(app).get('/api/health/services');
      expect(res.body.health.svc1.via).toBe('pylon');
      expect(res.body.health.svc1.status).toBe('healthy');
    });

    it('marks unhealthy when both direct and pylon fail (pylon configured)', async () => {
      const fetchT = jest.fn()
        .mockRejectedValueOnce(new Error('HEAD fail'))
        .mockRejectedValueOnce(new Error('GET fail'))
        .mockRejectedValueOnce(new Error('pylon fail'));
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ id: 'svc1', name: 'S1' }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const { app } = createApp({
        servicesStateManager: stateManager,
        fetchT,
        siteConfig: { tld: 'sami', pylon: { url: 'http://pylon.test' } },
      });
      const res = await request(app).get('/api/health/services');
      expect(res.body.health.svc1.status).toBe('unhealthy');
      expect(res.body.health.svc1.reason).toMatch(/direct \+ pylon/);
    });

    it('catches errors thrown by resolveServiceUrl and marks as error', async () => {
      resolveServiceUrl.mockImplementation(() => { throw new Error('resolver exploded'); });
      const stateManager = {
        read: jest.fn().mockResolvedValue([{ id: 'svc1', name: 'S1' }]),
        write: jest.fn(),
        update: jest.fn(),
      };
      const { app } = createApp({ servicesStateManager: stateManager });
      const res = await request(app).get('/api/health/services');
      expect(res.body.health.svc1.status).toBe('error');
      expect(res.body.health.svc1.reason).toMatch(/resolver exploded/);
      resolveServiceUrl.mockReturnValue('https://fallback.test');
    });
  });

  // ---- Coverage for /health-checks/incidents and history with pagination ----
  describe('GET /api/health-checks/incidents — non-empty', () => {
    it('returns incidents list', async () => {
      const healthChecker = {
        getCurrentStatus: jest.fn().mockReturnValue({}),
        getServiceStats: jest.fn(),
        configureService: jest.fn(),
        removeService: jest.fn(),
        getOpenIncidents: jest.fn().mockReturnValue([{ id: 'inc1', serviceId: 'svc1' }]),
        getIncidentHistory: jest.fn().mockReturnValue([]),
      };
      const { app } = createApp({ healthChecker });
      const res = await request(app).get('/api/health-checks/incidents');
      expect(res.status).toBe(200);
      expect(res.body.incidents).toHaveLength(1);
    });
  });
});