[glm-grade=A] fix(monitoring): DC-086 round-2 — probe/config race hardening + env parse + incident compare

Round-2 folds the judge-round fixes into DC-086:

- serviceGenerations map: checkService captures the config generation at
  entry and re-validates it before ANY state write (success + error
  paths). In-flight probes that resolve after removeService/updateService
  are discarded — deleted services can no longer resurrect status entries,
  fire incidents, or poke consecutiveFailures from beyond the grave.
- removeService now purges ALL per-service state: displayedStatus,
  consecutiveSinceChange, consecutiveFailures, pending backoff timers,
  and the serviceTimers entry (leaked a live setTimeout before).
- readPositiveIntEnv(): HEALTH_DOWN_THRESHOLD / HEALTH_UP_THRESHOLD
  parsing hardened — empty, non-numeric, fractional, zero, and negative
  values all fall back to defaults instead of Math.max(1, NaN)=NaN.
- previousStatus is captured BEFORE recordStatus() writes the new probe,
  so checkForIncidents() compares against the true prior state instead
  of the just-overwritten one (latent incident-suppression bug).
- Same-status hysteresis path returns the raw consistent snapshot
  (not the stale displayed one) so timestamps stay current without
  mixing contradictory fields.
- Tests: +14 (86 total across the two suites). New coverage: streak
  reset on agreement, malformed env fallbacks (each.of not-a-number/0/
  -2/1.5), in-flight probe after removeService does not resurrect state,
  getCurrentStatus serves internally-consistent displayed snapshot while
  raw currentStatus keeps the suppressed failure. Full suite 2601/2601.

Judge: GLM-5.3 cold-read via delegate_task (deleg_c9fd5900 task-0),
grade A round 1, zero blocking issues. Verdict URN:
urn:ump:quhs33ph2hhmsxjti63eg3ro4aiy34r6nws7z66ofk3bg3rcb3ca
(Codex primary quota-walled until 2026-08-29; GLM-4.6 direct 401;
stand-in chain per codex-as-judge SKILL.md, Sami 2026-08-17.)

dashcaddy-api/__tests__/health-checker-hysteresis.test.js

@@ -22,11 +22,17 @@ process.env.HEALTH_DATA_DIR = tmpDir;
 process.env.HEALTH_CONFIG_FILE = path.join(tmpDir, 'health-config.json');
 process.env.HEALTH_HISTORY_FILE = path.join(tmpDir, 'health-history.json');
 
-const { HealthChecker } = require('../src/monitoring/health-checker');
 // Module exports a singleton instance, not a class — see module.exports in
 // src/monitoring/health-checker.js. The test creates fresh state by replacing
 // the relevant maps on the singleton in beforeEach.
 const healthCheckerSingleton = require('../src/monitoring/health-checker');
+const originalDownThreshold = process.env.HEALTH_DOWN_THRESHOLD;
+const originalUpThreshold = process.env.HEALTH_UP_THRESHOLD;
+
+function restoreEnv(name, value) {
+  if (value === undefined) delete process.env[name];
+  else process.env[name] = value;
+}
 
 function makeUp(serviceId = 'svc1') {
   return {
@@ -68,6 +74,15 @@ describe('DC-086: hysteresis on the dashboard badge', () => {
     hc = healthCheckerSingleton;
   });
 
+  afterEach(() => {
+    restoreEnv('HEALTH_DOWN_THRESHOLD', originalDownThreshold);
+    restoreEnv('HEALTH_UP_THRESHOLD', originalUpThreshold);
+  });
+
+  afterAll(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
   test('first probe (no prior state) emits', () => {
     hc.recordStatus('svc1', makeUp());
     expect(emitSpy).toHaveBeenCalledTimes(1);
@@ -88,6 +103,19 @@ describe('DC-086: hysteresis on the dashboard badge', () => {
     expect(hc.displayedStatus.get('svc1').status).toBe('up');
   });
 
+  test('up, down, up, down, down resets the first streak before flipping', () => {
+    hc.recordStatus('svc1', makeUp());
+    hc.recordStatus('svc1', makeDown());
+    hc.recordStatus('svc1', makeUp());
+    hc.recordStatus('svc1', makeDown());
+    expect(hc.displayedStatus.get('svc1').status).toBe('up');
+    expect(emitSpy).toHaveBeenCalledTimes(1);
+
+    hc.recordStatus('svc1', makeDown());
+    expect(hc.displayedStatus.get('svc1').status).toBe('down');
+    expect(emitSpy).toHaveBeenCalledTimes(2);
+  });
+
   test('two consecutive "down" probes flip the badge to red', () => {
     hc.recordStatus('svc1', makeUp());        // baseline: green
     hc.recordStatus('svc1', makeDown());      // blip #1 — keep green (counter=1)
@@ -120,10 +148,28 @@ describe('DC-086: hysteresis on the dashboard badge', () => {
   });
 
   test('getCurrentStatus returns the displayed status, not the raw probe', () => {
-    hc.recordStatus('svc1', makeUp());
-    hc.recordStatus('svc1', makeDown());      // raw=down, displayed=up
+    const displayedUp = makeUp();
+    displayedUp.timestamp = '2026-08-22T09:59:00.000Z';
+    displayedUp.statusCode = 200;
+    displayedUp.message = 'healthy';
+    displayedUp.details = { source: 'accepted-up' };
+    hc.recordStatus('svc1', displayedUp);
+    const latestRaw = makeDown();
+    latestRaw.timestamp = '2026-08-22T10:00:00.000Z';
+    latestRaw.responseTime = 987;
+    latestRaw.statusCode = 500;
+    latestRaw.message = 'failed probe';
+    latestRaw.error = 'upstream failure';
+    latestRaw.details = { source: 'suppressed-down' };
+    hc.recordStatus('svc1', latestRaw);       // raw=down, displayed=up
     const out = hc.getCurrentStatus();
     expect(out['svc1'].status).toBe('up');    // shown to API consumers
+    expect(out['svc1'].timestamp).toBe(displayedUp.timestamp);
+    expect(out['svc1'].statusCode).toBe(200);
+    expect(out['svc1'].message).toBe('healthy');
+    expect(out['svc1'].error).toBeUndefined();
+    expect(out['svc1'].details).toEqual({ source: 'accepted-up' });
+    expect(hc.currentStatus.get('svc1')).toBe(latestRaw);
   });
 
   test('a long steady-green run produces exactly ONE emit (no per-probe spam)', () => {
@@ -142,7 +188,6 @@ describe('DC-086: hysteresis on the dashboard badge', () => {
   });
 
   test('DOWN_THRESHOLD env var is honored', () => {
-    const origDown = process.env.HEALTH_DOWN_THRESHOLD;
     process.env.HEALTH_DOWN_THRESHOLD = '3';
     jest.resetModules();
     const HC2Module = require('../src/monitoring/health-checker');
@@ -164,7 +209,89 @@ describe('DC-086: hysteresis on the dashboard badge', () => {
     hc2.recordStatus('svc1', makeDown());      // 3 — flip
     expect(spy).toHaveBeenCalledTimes(2);
     expect(hc2.displayedStatus.get('svc1').status).toBe('down');
-    if (origDown === undefined) delete process.env.HEALTH_DOWN_THRESHOLD;
-    else process.env.HEALTH_DOWN_THRESHOLD = origDown;
   });
-});
+
+  test.each(['not-a-number', '0', '-2', '1.5'])('malformed DOWN_THRESHOLD %s falls back to 2', value => {
+    process.env.HEALTH_DOWN_THRESHOLD = value;
+    jest.resetModules();
+    const hc2 = require('../src/monitoring/health-checker');
+    hc2.displayedStatus = new Map();
+    hc2.consecutiveSinceChange = new Map();
+    hc2.currentStatus = new Map();
+    hc2.history = {};
+    hc2.removeAllListeners('status-check');
+    const spy = jest.fn();
+    hc2.on('status-check', spy);
+    hc2.recordStatus('svc1', makeUp());
+    hc2.recordStatus('svc1', makeDown());
+    expect(spy).toHaveBeenCalledTimes(1);
+    hc2.recordStatus('svc1', makeDown());
+    expect(spy).toHaveBeenCalledTimes(2);
+  });
+
+  test('UP_THRESHOLD env var greater than 1 is honored', () => {
+    process.env.HEALTH_UP_THRESHOLD = '2';
+    jest.resetModules();
+    const hc2 = require('../src/monitoring/health-checker');
+    hc2.displayedStatus = new Map();
+    hc2.consecutiveSinceChange = new Map();
+    hc2.currentStatus = new Map();
+    hc2.history = {};
+    hc2.removeAllListeners('status-check');
+    const spy = jest.fn();
+    hc2.on('status-check', spy);
+    hc2.recordStatus('svc1', makeDown());
+    hc2.recordStatus('svc1', makeUp());
+    expect(spy).toHaveBeenCalledTimes(1);
+    expect(hc2.displayedStatus.get('svc1').status).toBe('down');
+    hc2.recordStatus('svc1', makeUp());
+    expect(spy).toHaveBeenCalledTimes(2);
+    expect(hc2.displayedStatus.get('svc1').status).toBe('up');
+  });
+
+  test.each(['not-a-number', '0', '-2', '1.5'])('malformed UP_THRESHOLD %s falls back to 1', value => {
+    process.env.HEALTH_UP_THRESHOLD = value;
+    jest.resetModules();
+    const hc2 = require('../src/monitoring/health-checker');
+    hc2.displayedStatus = new Map();
+    hc2.consecutiveSinceChange = new Map();
+    hc2.currentStatus = new Map();
+    hc2.history = {};
+    hc2.removeAllListeners('status-check');
+    const spy = jest.fn();
+    hc2.on('status-check', spy);
+    hc2.recordStatus('svc1', makeDown());
+    hc2.recordStatus('svc1', makeUp());
+    expect(spy).toHaveBeenCalledTimes(2);
+    expect(hc2.displayedStatus.get('svc1').status).toBe('up');
+  });
+
+  test('removeService clears hysteresis state before the same ID is re-added', () => {
+    hc.config.services.svc1 = { name: 'Service 1' };
+    hc.recordStatus('svc1', makeUp());
+    hc.recordStatus('svc1', makeDown());
+    expect(hc.displayedStatus.has('svc1')).toBe(true);
+    expect(hc.consecutiveSinceChange.get('svc1')).toBe(1);
+    hc.consecutiveFailures.set('svc1', 3);
+    const timer = setTimeout(() => {}, 60_000);
+    hc.serviceTimers.set('svc1', timer);
+
+    hc.saveConfig = jest.fn();
+    hc.removeService('svc1');
+
+    expect(hc.displayedStatus.has('svc1')).toBe(false);
+    expect(hc.consecutiveSinceChange.has('svc1')).toBe(false);
+    expect(hc.currentStatus.has('svc1')).toBe(false);
+    expect(hc.consecutiveFailures.has('svc1')).toBe(false);
+    expect(hc.serviceTimers.has('svc1')).toBe(false);
+
+    hc.config.services.svc1 = { name: 'Service 1 re-added' };
+    const emitSpyAfterReAdd = jest.fn();
+    hc.on('status-check', emitSpyAfterReAdd);
+    hc.recordStatus('svc1', makeDown());
+
+    expect(emitSpyAfterReAdd).toHaveBeenCalledTimes(1);
+    expect(hc.displayedStatus.get('svc1').status).toBe('down');
+    expect(hc.consecutiveSinceChange.has('svc1')).toBe(false);
+  });
+});

dashcaddy-api/__tests__/health-checker.test.js

@@ -203,6 +203,48 @@ describe('HealthChecker', () => {
       expect(result.error).toBe('ECONNREFUSED');
     });
 
+    it('opens and resolves an outage incident across real checkService transitions', async () => {
+      healthChecker._doRequest = jest.fn()
+        .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} })
+        .mockResolvedValueOnce({ healthy: false, statusCode: 500, message: 'down', details: {} })
+        .mockResolvedValueOnce({ healthy: true, statusCode: 200, message: 'ok', details: {} });
+
+      const config = { url: 'http://test.local' };
+      await healthChecker.checkService('svc1', config);
+      await healthChecker.checkService('svc1', config);
+
+      expect(healthChecker.incidents).toHaveLength(1);
+      expect(healthChecker.incidents[0]).toMatchObject({
+        serviceId: 'svc1',
+        type: 'outage',
+        status: 'open'
+      });
+
+      await healthChecker.checkService('svc1', config);
+      expect(healthChecker.incidents[0].status).toBe('resolved');
+      expect(healthChecker.incidents[0].resolvedAt).toBeDefined();
+    });
+
+    it('does not resurrect state when an in-flight probe resolves after removal', async () => {
+      let resolveProbe;
+      healthChecker.config.services.svc1 = { url: 'http://test.local' };
+      healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
+        resolveProbe = resolve;
+      }));
+
+      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
+      healthChecker.saveConfig = jest.fn();
+      healthChecker.removeService('svc1');
+      resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
+      await pending;
+
+      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
+      expect(healthChecker.displayedStatus.has('svc1')).toBe(false);
+      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
+      expect(healthChecker.history.svc1).toBeUndefined();
+      expect(healthChecker.incidents).toEqual([]);
+    });
+
     it('increments consecutive failures on error', async () => {
       healthChecker._doRequest = jest.fn().mockRejectedValue(new Error('fail'));