dashcaddy/dashcaddy-api/__tests__/health-checker.test.js

jest.mock('fs', () => ({
  existsSync: jest.fn().mockReturnValue(false),
  readFileSync: jest.fn().mockReturnValue('{"services":{}}'),
  writeFileSync: jest.fn(),
}));

jest.useFakeTimers();

describe('HealthChecker', () => {
  let HealthChecker, healthChecker, fs;

  beforeEach(() => {
    jest.resetModules();
    fs = require('fs');
    fs.existsSync.mockReturnValue(false);
    fs.readFileSync.mockReturnValue('{"services":{}}');
    fs.writeFileSync.mockImplementation(() => {});

    // Fresh instance each test
    HealthChecker = require('../src/monitoring/health-checker').constructor;
    healthChecker = new HealthChecker();
  });

  afterEach(() => {
    healthChecker.stop();
    jest.clearAllTimers();
  });

  describe('constructor', () => {
    it('initializes with empty state', () => {
      expect(healthChecker.currentStatus).toBeInstanceOf(Map);
      expect(healthChecker.incidents).toEqual([]);
      expect(healthChecker.checking).toBe(false);
    });

    it('loads config from file when it exists', () => {
      jest.resetModules();
      fs = require('fs');
      fs.existsSync.mockReturnValue(true);
      fs.readFileSync.mockReturnValue(JSON.stringify({
        services: { svc1: { url: 'http://test.local', enabled: true } }
      }));

      HealthChecker = require('../src/monitoring/health-checker').constructor;
      const hc = new HealthChecker();
      expect(hc.config.services.svc1).toBeDefined();
    });

    it('returns default config on parse error', () => {
      jest.resetModules();
      fs = require('fs');
      fs.existsSync.mockReturnValue(true);
      fs.readFileSync.mockReturnValue('invalid json');

      HealthChecker = require('../src/monitoring/health-checker').constructor;
      const hc = new HealthChecker();
      expect(hc.config).toEqual({ services: {} });
    });
  });

  describe('start / stop', () => {
    it('start sets checking to true and schedules interval', () => {
      // Mock checkAll to prevent real HTTP calls
      healthChecker.checkAll = jest.fn();
      healthChecker.start();
      expect(healthChecker.checking).toBe(true);
      expect(healthChecker.checkAll).toHaveBeenCalled();
    });

    it('start is idempotent (no-op if already checking)', () => {
      healthChecker.checkAll = jest.fn();
      healthChecker.start();
      healthChecker.start(); // second call
      expect(healthChecker.checkAll).toHaveBeenCalledTimes(1);
    });

    it('stop clears interval and resets state', () => {
      healthChecker.checkAll = jest.fn();
      healthChecker.start();
      healthChecker.stop();
      expect(healthChecker.checking).toBe(false);
      expect(healthChecker.checkInterval).toBeNull();
    });

    it('stop is idempotent (no-op if not checking)', () => {
      healthChecker.stop(); // should not throw
      expect(healthChecker.checking).toBe(false);
    });
  });

  describe('getBackoffInterval', () => {
    it('returns base interval when no failures', () => {
      const interval = healthChecker.getBackoffInterval('svc1');
      expect(interval).toBe(30000); // CHECK_INTERVAL default
    });

    it('doubles interval per consecutive failure', () => {
      healthChecker.consecutiveFailures.set('svc1', 1);
      expect(healthChecker.getBackoffInterval('svc1')).toBe(60000);

      healthChecker.consecutiveFailures.set('svc1', 2);
      expect(healthChecker.getBackoffInterval('svc1')).toBe(120000);
    });

    it('caps at MAX_CHECK_INTERVAL', () => {
      healthChecker.consecutiveFailures.set('svc1', 100);
      expect(healthChecker.getBackoffInterval('svc1')).toBe(300000);
    });
  });

  describe('evaluateHealth', () => {
    it('returns true for expected status code', () => {
      const result = healthChecker.evaluateHealth(200, '', { expectedStatusCodes: [200] });
      expect(result).toBe(true);
    });

    it('returns false for unexpected status code', () => {
      const result = healthChecker.evaluateHealth(500, '', { expectedStatusCodes: [200] });
      expect(result).toBe(false);
    });

    it('defaults to accepting common 2xx/3xx codes', () => {
      expect(healthChecker.evaluateHealth(200, '', {})).toBe(true);
      expect(healthChecker.evaluateHealth(301, '', {})).toBe(true);
      expect(healthChecker.evaluateHealth(500, '', {})).toBe(false);
    });

    it('defaults to accepting 401/403 (auth-walled UIs still prove the service is up)', () => {
      expect(healthChecker.evaluateHealth(401, '', {})).toBe(true);
      expect(healthChecker.evaluateHealth(403, '', {})).toBe(true);
    });

    it('defaults to accepting 429 (rate-limited upstream is still reachable)', () => {
      // The upstream answered — it just throttled us. Failing the check here
      // caused the authLimiter feedback loop (DC-XXX) where every gated
      // service flipped red after 20 probes / 15 min.
      expect(healthChecker.evaluateHealth(429, '', {})).toBe(true);
    });

    it('checks body pattern with regex', () => {
      const config = { expectedBodyPattern: 'ok|healthy' };
      expect(healthChecker.evaluateHealth(200, 'status: ok', config)).toBe(true);
      expect(healthChecker.evaluateHealth(200, 'status: error', config)).toBe(false);
    });

    it('checks body contains text', () => {
      const config = { expectedBodyContains: 'alive' };
      expect(healthChecker.evaluateHealth(200, 'I am alive!', config)).toBe(true);
      expect(healthChecker.evaluateHealth(200, 'dead', config)).toBe(false);
    });
  });

  describe('recordStatus', () => {
    it('updates currentStatus map', () => {
      const status = { serviceId: 'svc1', status: 'up', timestamp: new Date().toISOString() };
      healthChecker.recordStatus('svc1', status);
      expect(healthChecker.currentStatus.get('svc1')).toEqual(status);
    });

    it('appends to history', () => {
      const status1 = { serviceId: 'svc1', status: 'up', timestamp: new Date().toISOString() };
      const status2 = { serviceId: 'svc1', status: 'down', timestamp: new Date().toISOString() };
      healthChecker.recordStatus('svc1', status1);
      healthChecker.recordStatus('svc1', status2);
      expect(healthChecker.history['svc1']).toHaveLength(2);
    });

    it('emits status-check event', () => {
      const handler = jest.fn();
      healthChecker.on('status-check', handler);
      const status = { serviceId: 'svc1', status: 'up' };
      healthChecker.recordStatus('svc1', status);
      expect(handler).toHaveBeenCalledWith(status);
    });
  });

  describe('checkService', () => {
    it('returns up status on successful health check', async () => {
      healthChecker._doRequest = jest.fn().mockResolvedValue({
        healthy: true, statusCode: 200, message: 'Service is healthy', details: {}
      });

      const config = { url: 'http://test.local' };
      const result = await healthChecker.checkService('svc1', config);
      expect(result.status).toBe('up');
      expect(result.serviceId).toBe('svc1');
    });

    it('returns down status on failed health check', async () => {
      healthChecker._doRequest = jest.fn().mockResolvedValue({
        healthy: false, statusCode: 500, message: 'fail', details: {}
      });

      const result = await healthChecker.checkService('svc1', { url: 'http://test.local' });
      expect(result.status).toBe('down');
    });

    it('returns down status on request error', async () => {
      healthChecker._doRequest = jest.fn().mockRejectedValue(new Error('ECONNREFUSED'));

      const result = await healthChecker.checkService('svc1', { url: 'http://test.local' });
      expect(result.status).toBe('down');
      expect(result.error).toBe('ECONNREFUSED');
    });

    it('opens and resolves an outage incident across real checkService transitions', async () => {
      healthChecker._doRequest = jest.fn()
        .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
        .mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
        .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });

      const config = { url: 'http://test.local' };
      await healthChecker.checkService('svc1', config);
      await healthChecker.checkService('svc1', config);

      expect(healthChecker.incidents).toHaveLength(1);
      expect(healthChecker.incidents[0]).toMatchObject({
        serviceId: 'svc1',
        type: 'outage',
        status: 'open'
      });

      await healthChecker.checkService('svc1', config);
      expect(healthChecker.incidents[0].status).toBe('resolved');
      expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
    });

    it('does not resurrect state when an in-flight probe resolves after removal', async () => {
      let resolveProbe;
      healthChecker.config.services.svc1 = { url: 'http://test.local' };
      healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
        resolveProbe = resolve;
      }));

      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
      healthChecker.saveConfig = jest.fn();
      healthChecker.removeService('svc1');
      resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
      await pending;

      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
      expect(healthChecker.displayedStatus.has('svc1')).toBe(false);
      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
      expect(healthChecker.history.svc1).toBeUndefined();
      expect(healthChecker.incidents).toEqual([]);
    });

    it('increments consecutive failures on error', async () => {
      healthChecker._doRequest = jest.fn().mockRejectedValue(new Error('fail'));

      await healthChecker.checkService('svc1', { url: 'http://test.local' });
      expect(healthChecker.consecutiveFailures.get('svc1')).toBe(1);

      await healthChecker.checkService('svc1', { url: 'http://test.local' });
      expect(healthChecker.consecutiveFailures.get('svc1')).toBe(2);
    });

    it('clears consecutive failures on success', async () => {
      healthChecker.consecutiveFailures.set('svc1', 5);
      healthChecker._doRequest = jest.fn().mockResolvedValue({
        healthy: true, statusCode: 200, message: 'ok', details: {}
      });

      await healthChecker.checkService('svc1', { url: 'http://test.local' });
      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
    });
  });

  describe('performHealthCheck', () => {
    it('falls back to GET when HEAD returns 501', async () => {
      healthChecker._doRequest = jest.fn()
        .mockResolvedValueOnce({ statusCode: 501 })
        .mockResolvedValueOnce({ healthy: true, statusCode: 200 });

      const result = await healthChecker.performHealthCheck({ url: 'http://test.local', method: 'HEAD' });
      expect(healthChecker._doRequest).toHaveBeenCalledTimes(2);
      expect(result.statusCode).toBe(200);
    });

    it('falls back to GET when HEAD returns 405', async () => {
      healthChecker._doRequest = jest.fn()
        .mockResolvedValueOnce({ statusCode: 405 })
        .mockResolvedValueOnce({ healthy: true, statusCode: 200 });

      const result = await healthChecker.performHealthCheck({ url: 'http://test.local', method: 'HEAD' });
      expect(result.statusCode).toBe(200);
    });

    it('does not fallback for GET requests returning 501', async () => {
      healthChecker._doRequest = jest.fn()
        .mockResolvedValueOnce({ statusCode: 501, healthy: false });

      const result = await healthChecker.performHealthCheck({ url: 'http://test.local' });
      expect(healthChecker._doRequest).toHaveBeenCalledTimes(1);
    });
  });

  describe('_doRequest header injection', () => {
    // Verifies the X-DashCaddy-HealthCheck marker header is set on every
    // outgoing probe. Caddy uses this header (combined with a trusted source
    // IP) to bypass forward_auth for probes from the local container, which
    // is what stops the authLimiter feedback loop on gated services.
    // CI doesn't make real network calls — we capture the options object
    // via a tiny http mock and assert on it.
    //
    // Note: the suite runs under jest.useFakeTimers(), so we cannot rely on
    // setImmediate / setTimeout to fire the fake response. We emit 'end'
    // synchronously after attaching listeners, which the response handler
    // in _doRequest will receive on the same tick.
    it('sends X-DashCaddy-HealthCheck: 1 on every probe', () => {
      const https = require('https');
      const { EventEmitter } = require('events');
      const original = https.request;
      let capturedOptions = null;
      https.request = (options, cb) => {
        capturedOptions = options;
        const fakeRes = new EventEmitter();
        fakeRes.statusCode = 200;
        fakeRes.headers = {};
        // Call cb synchronously so listeners attach BEFORE we emit 'end'.
        cb(fakeRes);
        fakeRes.emit('end');
        const fakeReq = new EventEmitter();
        fakeReq.end = () => {};
        fakeReq.write = () => {};
        fakeReq.destroy = () => {};
        return fakeReq;
      };

      try {
        return healthChecker._doRequest({ url: 'https://example.sami/test', method: 'HEAD' }, 'HEAD').then(() => {
          expect(capturedOptions).not.toBeNull();
          expect(capturedOptions.headers['X-DashCaddy-HealthCheck']).toBe('1');
        });
      } finally {
        https.request = original;
      }
    });

    it('preserves user-supplied headers while adding the marker', () => {
      const https = require('https');
      const { EventEmitter } = require('events');
      const original = https.request;
      let capturedOptions = null;
      https.request = (options, cb) => {
        capturedOptions = options;
        const fakeRes = new EventEmitter();
        fakeRes.statusCode = 200;
        fakeRes.headers = {};
        cb(fakeRes);
        fakeRes.emit('end');
        const fakeReq = new EventEmitter();
        fakeReq.end = () => {};
        fakeReq.write = () => {};
        fakeReq.destroy = () => {};
        return fakeReq;
      };

      try {
        return healthChecker._doRequest({
          url: 'https://example.sami/test',
          method: 'GET',
          headers: { 'User-Agent': 'DashCaddy-Test/1.0', 'X-Custom': 'foo' }
        }, 'GET').then(() => {
          expect(capturedOptions.headers['X-DashCaddy-HealthCheck']).toBe('1');
          expect(capturedOptions.headers['User-Agent']).toBe('DashCaddy-Test/1.0');
          expect(capturedOptions.headers['X-Custom']).toBe('foo');
        });
      } finally {
        https.request = original;
      }
    });
  });

  describe('incidents', () => {
    it('createIncident adds a new incident', () => {
      const status = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'Service down', status);
      expect(healthChecker.incidents).toHaveLength(1);
      expect(healthChecker.incidents[0].serviceId).toBe('svc1');
      expect(healthChecker.incidents[0].type).toBe('outage');
      expect(healthChecker.incidents[0].status).toBe('open');
    });

    it('createIncident increments existing open incident', () => {
      const status = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'down', status);
      healthChecker.createIncident('svc1', 'outage', 'still down', status);
      expect(healthChecker.incidents).toHaveLength(1);
      expect(healthChecker.incidents[0].occurrences).toBe(2);
    });

    it('resolveIncident sets status to resolved', () => {
      const status = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'down', status);
      healthChecker.resolveIncident('svc1', 'outage', status);
      expect(healthChecker.incidents[0].status).toBe('resolved');
      expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
    });

    it('resolveIncident is no-op for non-existent incidents', () => {
      const status = { timestamp: new Date().toISOString() };
      healthChecker.resolveIncident('svc1', 'outage', status);
      expect(healthChecker.incidents).toHaveLength(0);
    });

    it('getOpenIncidents filters resolved', () => {
      const ts = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'down', ts);
      healthChecker.createIncident('svc2', 'slow-response', 'slow', ts);
      healthChecker.resolveIncident('svc1', 'outage', ts);

      const open = healthChecker.getOpenIncidents();
      expect(open).toHaveLength(1);
      expect(open[0].serviceId).toBe('svc2');
    });

    it('getIncidentHistory returns recent incidents in reverse order', () => {
      const ts = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'first', ts);
      healthChecker.createIncident('svc2', 'outage', 'second', ts);

      const history = healthChecker.getIncidentHistory();
      expect(history[0].serviceId).toBe('svc2');
      expect(history[1].serviceId).toBe('svc1');
    });

    it('emits incident-created event', () => {
      const handler = jest.fn();
      healthChecker.on('incident-created', handler);
      healthChecker.createIncident('svc1', 'outage', 'down', { timestamp: new Date().toISOString() });
      expect(handler).toHaveBeenCalled();
    });

    it('emits incident-resolved event', () => {
      const handler = jest.fn();
      healthChecker.on('incident-resolved', handler);
      const ts = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'down', ts);
      healthChecker.resolveIncident('svc1', 'outage', ts);
      expect(handler).toHaveBeenCalled();
    });
  });

  describe('calculateSeverity', () => {
    it('returns critical for outage', () => {
      expect(healthChecker.calculateSeverity('outage')).toBe('critical');
    });
    it('returns high for sla-violation', () => {
      expect(healthChecker.calculateSeverity('sla-violation')).toBe('high');
    });
    it('returns medium for slow-response', () => {
      expect(healthChecker.calculateSeverity('slow-response')).toBe('medium');
    });
    it('returns low for unknown', () => {
      expect(healthChecker.calculateSeverity('unknown')).toBe('low');
    });
  });

  describe('checkForIncidents', () => {
    it('creates outage incident on status change up -> down', () => {
      // Simulate previous up status
      healthChecker.currentStatus.set('svc1', { status: 'up' });
      const status = { status: 'down', timestamp: new Date().toISOString(), responseTime: 100 };
      healthChecker.checkForIncidents('svc1', status, {});
      expect(healthChecker.incidents).toHaveLength(1);
      expect(healthChecker.incidents[0].type).toBe('outage');
    });

    it('resolves outage incident on status change down -> up', () => {
      healthChecker.currentStatus.set('svc1', { status: 'down' });
      const ts = { timestamp: new Date().toISOString() };
      healthChecker.createIncident('svc1', 'outage', 'was down', ts);

      const status = { status: 'up', timestamp: new Date().toISOString(), responseTime: 100 };
      healthChecker.checkForIncidents('svc1', status, {});
      expect(healthChecker.incidents[0].status).toBe('resolved');
    });

    it('creates slow-response incident when exceeding threshold', () => {
      const status = { status: 'up', timestamp: new Date().toISOString(), responseTime: 6000 };
      healthChecker.checkForIncidents('svc1', status, { slowResponseThreshold: 5000 });
      expect(healthChecker.incidents.some(i => i.type === 'slow-response')).toBe(true);
    });
  });

  describe('uptime and stats', () => {
    beforeEach(() => {
      const now = Date.now();
      healthChecker.history['svc1'] = [
        { status: 'up', responseTime: 100, timestamp: new Date(now - 3600000).toISOString() },
        { status: 'up', responseTime: 200, timestamp: new Date(now - 1800000).toISOString() },
        { status: 'down', responseTime: 5000, timestamp: new Date(now - 900000).toISOString() },
        { status: 'up', responseTime: 150, timestamp: new Date(now - 60000).toISOString() },
      ];
    });

    it('calculateUptime returns correct percentage', () => {
      const uptime = healthChecker.calculateUptime('svc1', 24);
      expect(uptime).toBe(75); // 3 out of 4 checks up
    });

    it('calculateUptime returns 100 for unknown service', () => {
      expect(healthChecker.calculateUptime('unknown', 24)).toBe(100);
    });

    it('calculateAverageResponseTime returns correct average', () => {
      const avg = healthChecker.calculateAverageResponseTime('svc1', 24);
      expect(avg).toBe((100 + 200 + 5000 + 150) / 4);
    });

    it('calculateAverageResponseTime returns 0 for unknown service', () => {
      expect(healthChecker.calculateAverageResponseTime('unknown', 24)).toBe(0);
    });

    it('getServiceHistory filters by time period', () => {
      const history = healthChecker.getServiceHistory('svc1', 24);
      expect(history.length).toBe(4);

      // Very short period should exclude older entries
      const recent = healthChecker.getServiceHistory('svc1', 0.01); // ~36 seconds
      expect(recent.length).toBeLessThan(4);
    });

    it('getServiceStats returns null for unknown service', () => {
      expect(healthChecker.getServiceStats('unknown')).toBeNull();
    });

    it('getServiceStats returns correct stats', () => {
      const stats = healthChecker.getServiceStats('svc1', 24);
      expect(stats.totalChecks).toBe(4);
      expect(stats.upChecks).toBe(3);
      expect(stats.downChecks).toBe(1);
      expect(stats.uptime).toBe(75);
      expect(stats.responseTime.min).toBe(100);
      expect(stats.responseTime.max).toBe(5000);
    });
  });

  describe('calculatePercentile', () => {
    it('returns correct p95', () => {
      const values = Array.from({ length: 100 }, (_, i) => i + 1);
      const p95 = healthChecker.calculatePercentile(values, 95);
      expect(p95).toBe(95);
    });

    it('returns 0 for empty array', () => {
      expect(healthChecker.calculatePercentile([], 95)).toBe(0);
    });
  });

  describe('getCurrentStatus', () => {
    it('returns enriched status for all services', () => {
      healthChecker.config.services = {
        svc1: { name: 'Test Service' }
      };
      healthChecker.currentStatus.set('svc1', {
        status: 'up', responseTime: 100, timestamp: new Date().toISOString()
      });

      const result = healthChecker.getCurrentStatus();
      expect(result.svc1).toBeDefined();
      expect(result.svc1.name).toBe('Test Service');
      expect(result.svc1.uptime).toBeDefined();
      expect(result.svc1.uptime['24h']).toBeDefined();
    });
  });

  describe('configureService / removeService', () => {
    it('configureService saves config to file', () => {
      healthChecker.configureService('svc1', {
        name: 'My Service',
        url: 'http://localhost:3000',
        timeout: 10000
      });

      expect(healthChecker.config.services.svc1).toBeDefined();
      expect(healthChecker.config.services.svc1.url).toBe('http://localhost:3000');
      expect(fs.writeFileSync).toHaveBeenCalled();
    });

    it('removeService cleans up all traces', () => {
      healthChecker.configureService('svc1', { url: 'http://test.local' });
      healthChecker.currentStatus.set('svc1', { status: 'up' });
      healthChecker.history['svc1'] = [{ status: 'up' }];

      healthChecker.removeService('svc1');
      expect(healthChecker.config.services.svc1).toBeUndefined();
      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
      expect(healthChecker.history['svc1']).toBeUndefined();
    });
  });

  describe('DC-088: removeService generation tombstones + incident closure', () => {
    it('does not leak a serviceGenerations entry and records a tombstone', () => {
      healthChecker.configureService('svc1', { url: 'http://test.local' });
      expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);

      healthChecker.removeService('svc1');

      expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
      const tomb = healthChecker.removedGenerations.get('svc1');
      expect(tomb).toBeDefined();
      expect(tomb.generation).toBeGreaterThan(0);
      expect(tomb.removedAt).toBeGreaterThan(0);
    });

    it('re-added service gets a strictly higher generation (no ABA)', () => {
      healthChecker.configureService('svc1', { url: 'http://test.local' });
      const gen1 = healthChecker.serviceGenerations.get('svc1');

      healthChecker.removeService('svc1');
      healthChecker.configureService('svc1', { url: 'http://test.local/v2' });

      const gen2 = healthChecker.serviceGenerations.get('svc1');
      expect(gen2).toBeGreaterThan(gen1);
      expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
    });

    it('closes open incidents for the removed service as resolved', () => {
      healthChecker.saveConfig = jest.fn();
      healthChecker.incidents.push({
        id: 'incident-test-1',
        serviceId: 'svc1',
        type: 'outage',
        status: 'open',
        createdAt: new Date(Date.now() - 60_000).toISOString()
      });
      healthChecker.incidents.push({
        id: 'incident-other',
        serviceId: 'svc2',
        type: 'outage',
        status: 'open',
        createdAt: new Date(Date.now() - 60_000).toISOString()
      });
      const resolvedSpy = jest.fn();
      healthChecker.on('incident-resolved', resolvedSpy);

      healthChecker.removeService('svc1');

      const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
      expect(closed.status).toBe('resolved');
      expect(closed.resolvedBy).toBe('service-removed');
      expect(closed.resolvedAt).toBeDefined();
      expect(closed.duration).toBeGreaterThan(0);
      expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
      expect(resolvedSpy).toHaveBeenCalledTimes(1);
    });

    it('in-flight probe captured before removal is discarded via tombstone', async () => {
      let resolveProbe;
      healthChecker.config.services.svc1 = { url: 'http://test.local' };
      healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
        resolveProbe = resolve;
      }));

      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
      healthChecker.saveConfig = jest.fn();
      healthChecker.removeService('svc1');
      resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
      await pending;

      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
    });

    it('a rejected in-flight probe after removal does not re-create failure state', async () => {
      let rejectProbe;
      healthChecker.config.services.svc1 = { url: 'http://test.local' };
      healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
        rejectProbe = reject;
      }));

      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
      healthChecker.saveConfig = jest.fn();
      healthChecker.removeService('svc1');
      rejectProbe(new Error('late failure'));
      await pending;

      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
    });

    it('sweeps expired tombstones in cleanupHistory', () => {
      healthChecker.removedGenerations.set('svc1', {
        generation: 1,
        removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
      });
      healthChecker.removedGenerations.set('svc2', {
        generation: 2,
        removedAt: Date.now() // fresh
      });

      healthChecker.cleanupHistory();

      expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
      expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
    });
  });

  describe('cleanupHistory', () => {
    it('removes entries older than retention period', () => {
      const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago
      const recent = new Date().toISOString();
      healthChecker.history['svc1'] = [
        { timestamp: old, status: 'up' },
        { timestamp: recent, status: 'up' },
      ];

      healthChecker.cleanupHistory();
      expect(healthChecker.history['svc1']).toHaveLength(1);
      expect(healthChecker.history['svc1'][0].timestamp).toBe(recent);
    });
  });

  describe('loadConfig / saveConfig', () => {
    it('saveConfig writes JSON to file', () => {
      healthChecker.config = { services: { svc1: { url: 'http://test' } } };
      healthChecker.saveConfig();
      expect(fs.writeFileSync).toHaveBeenCalledWith(
        expect.any(String),
        expect.stringContaining('"svc1"')
      );
    });

    it('saveConfig handles write errors gracefully', () => {
      fs.writeFileSync.mockImplementation(() => { throw new Error('disk full'); });
      expect(() => healthChecker.saveConfig()).not.toThrow();
    });
  });

  describe('loadHistory / saveHistory', () => {
    it('loadHistory returns empty object when file missing', () => {
      const history = healthChecker.loadHistory();
      expect(history).toEqual({});
    });

    it('loadHistory parses JSON from file', () => {
      fs.existsSync.mockReturnValue(true);
      fs.readFileSync.mockReturnValue(JSON.stringify({ svc1: [{ status: 'up' }] }));
      const history = healthChecker.loadHistory();
      expect(history.svc1).toHaveLength(1);
    });

    it('saveHistory writes history to file', () => {
      healthChecker.history = { svc1: [{ status: 'up' }] };
      healthChecker.saveHistory();
      expect(fs.writeFileSync).toHaveBeenCalled();
    });
  });
});