Merge dc/DC-088-remove-service-tombstones: removeService generation tombstones + incident closure [glm-grade=B]

- serviceGenerations no longer leaks entries: removeService deletes the live
  entry and records a TTL'd (10min) tombstone swept by cleanupHistory
- monotonic instance-wide generationSeq prevents generation reuse across
  remove->re-add cycles (ABA) and supersedes tombstones on re-configure
- _isStaleCapture(): presence-aware stale check — live entry must match
  exactly; no entry is stale only under a higher-generation tombstone
  (preserves correct behavior for disk-loaded never-configured services)
- catch path increments consecutiveFailures only after the stale check, so
  a late-rejected probe cannot resurrect state for a removed service
- open incidents for a removed service close via the standard resolve path
  (resolvedBy=service-removed, WS/SSE incident-resolved broadcast)
- 6 regression tests; full suite 2608/2608 green

Judge: GLM-5.3 cold read (Codex stand-in), verdict B/ship, zero blockers

dashcaddy-api/__tests__/health-checker.test.js

@@ -591,6 +591,113 @@ describe('HealthChecker', () => {
     });
   });
 
+  describe('DC-088: removeService generation tombstones + incident closure', () => {
+    it('does not leak a serviceGenerations entry and records a tombstone', () => {
+      healthChecker.configureService('svc1', { url: 'http://test.local' });
+      expect(healthChecker.serviceGenerations.has('svc1')).toBe(true);
+
+      healthChecker.removeService('svc1');
+
+      expect(healthChecker.serviceGenerations.has('svc1')).toBe(false);
+      const tomb = healthChecker.removedGenerations.get('svc1');
+      expect(tomb).toBeDefined();
+      expect(tomb.generation).toBeGreaterThan(0);
+      expect(tomb.removedAt).toBeGreaterThan(0);
+    });
+
+    it('re-added service gets a strictly higher generation (no ABA)', () => {
+      healthChecker.configureService('svc1', { url: 'http://test.local' });
+      const gen1 = healthChecker.serviceGenerations.get('svc1');
+
+      healthChecker.removeService('svc1');
+      healthChecker.configureService('svc1', { url: 'http://test.local/v2' });
+
+      const gen2 = healthChecker.serviceGenerations.get('svc1');
+      expect(gen2).toBeGreaterThan(gen1);
+      expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
+    });
+
+    it('closes open incidents for the removed service as resolved', () => {
+      healthChecker.saveConfig = jest.fn();
+      healthChecker.incidents.push({
+        id: 'incident-test-1',
+        serviceId: 'svc1',
+        type: 'outage',
+        status: 'open',
+        createdAt: new Date(Date.now() - 60_000).toISOString()
+      });
+      healthChecker.incidents.push({
+        id: 'incident-other',
+        serviceId: 'svc2',
+        type: 'outage',
+        status: 'open',
+        createdAt: new Date(Date.now() - 60_000).toISOString()
+      });
+      const resolvedSpy = jest.fn();
+      healthChecker.on('incident-resolved', resolvedSpy);
+
+      healthChecker.removeService('svc1');
+
+      const closed = healthChecker.incidents.find(i => i.id === 'incident-test-1');
+      expect(closed.status).toBe('resolved');
+      expect(closed.resolvedBy).toBe('service-removed');
+      expect(closed.resolvedAt).toBeDefined();
+      expect(closed.duration).toBeGreaterThan(0);
+      expect(healthChecker.incidents.find(i => i.id === 'incident-other').status).toBe('open');
+      expect(resolvedSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('in-flight probe captured before removal is discarded via tombstone', async () => {
+      let resolveProbe;
+      healthChecker.config.services.svc1 = { url: 'http://test.local' };
+      healthChecker._doRequest = jest.fn(() => new Promise(resolve => {
+        resolveProbe = resolve;
+      }));
+
+      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
+      healthChecker.saveConfig = jest.fn();
+      healthChecker.removeService('svc1');
+      resolveProbe({ healthy: true, statusCode: 200, message: 'late', details: {} });
+      await pending;
+
+      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
+      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
+    });
+
+    it('a rejected in-flight probe after removal does not re-create failure state', async () => {
+      let rejectProbe;
+      healthChecker.config.services.svc1 = { url: 'http://test.local' };
+      healthChecker._doRequest = jest.fn(() => new Promise((resolve, reject) => {
+        rejectProbe = reject;
+      }));
+
+      const pending = healthChecker.checkService('svc1', healthChecker.config.services.svc1);
+      healthChecker.saveConfig = jest.fn();
+      healthChecker.removeService('svc1');
+      rejectProbe(new Error('late failure'));
+      await pending;
+
+      expect(healthChecker.consecutiveFailures.has('svc1')).toBe(false);
+      expect(healthChecker.currentStatus.has('svc1')).toBe(false);
+    });
+
+    it('sweeps expired tombstones in cleanupHistory', () => {
+      healthChecker.removedGenerations.set('svc1', {
+        generation: 1,
+        removedAt: Date.now() - 60 * 60 * 1000 // 1h ago, TTL default 10m
+      });
+      healthChecker.removedGenerations.set('svc2', {
+        generation: 2,
+        removedAt: Date.now() // fresh
+      });
+
+      healthChecker.cleanupHistory();
+
+      expect(healthChecker.removedGenerations.has('svc1')).toBe(false);
+      expect(healthChecker.removedGenerations.has('svc2')).toBe(true);
+    });
+  });
+
   describe('cleanupHistory', () => {
     it('removes entries older than retention period', () => {
       const old = new Date(Date.now() - 35 * 24 * 60 * 60 * 1000).toISOString(); // 35 days ago

dashcaddy-api/__tests__/health-endpoints.test.js

@@ -26,6 +26,19 @@ jest.mock('dockerode', () => {
 function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk = true } = {}) {
   process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
 
+  // DC-087 — mirror src/app.js faithfully: the caddy check goes through
+  // fetchT (which injects the Origin header Caddy's enforce_origin allowlist
+  // requires), and is MOCKED so the suite is hermetic — no live request to a
+  // real Caddy admin on :2019. The previous raw-`fetch` mirror sent an
+  // Origin-less probe to the LIVE admin whenever the full suite ran on the
+  // prod host (adversarial cron every 30 min): 12 journal 403 lines per run,
+  // ~700/day of `client is not allowed to access from origin ''` noise,
+  // plus a false checks.caddy.ok=false in the mirrored readiness payload.
+  const fetchT = jest.spyOn(require('../src/utils/http'), 'fetchT')
+    .mockImplementation(async () => (caddyOk
+      ? { ok: true, status: 200 }
+      : { ok: false, status: 403 }));
+
   const app = express();
   const config = {
     CONFIG_FILE: '/tmp/dc-test-config.json',
@@ -103,9 +116,13 @@ function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk
       allOk = false;
     }
 
+    // DC-087 — mirror src/app.js exactly (fetchT, not raw fetch). fetchT is
+    // mocked at buildApp() scope, so this stays hermetic: no live probe to a
+    // real Caddy admin (the old raw-fetch mirror 403-spammed the prod journal
+    // every time the adversarial cron ran the full suite on this host).
     try {
       const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
-      const response = await fetch(`${caddyUrl}/config/apps/http/servers/srv0/listen`, { signal: AbortSignal.timeout(10000) });
+      const response = await fetchT(`${caddyUrl}/config/apps/http/servers/srv0/listen`, {}, 10000);
       checks.caddy = { ok: response.ok, status: response.status };
       if (!response.ok) allOk = false;
     } catch (e) {

dashcaddy-api/__tests__/health-probe-aliases.test.js

@@ -33,9 +33,18 @@ jest.mock('dockerode', () => {
 
 // Mirror the canonical handler block from src/app.js — if this drifts from
 // the real handler, these tests will start failing and force a sync.
-function buildApp({ configOk = true, servicesOk = true, dockerOk = true } = {}) {
+function buildApp({ configOk = true, servicesOk = true, dockerOk = true, caddyOk = true } = {}) {
   process.env.MOCK_DOCKER_DOWN = dockerOk ? '0' : '1';
 
+  // DC-087 — mirror src/app.js: caddy check via fetchT (Origin-injecting),
+  // mocked here so the suite is hermetic. The old raw-fetch mirror probed the
+  // LIVE Caddy admin on :2019 whenever the full suite ran on the prod host
+  // (adversarial cron): Origin-less → 403 → 12 journal error lines per run.
+  const fetchT = jest.spyOn(require('../src/utils/http'), 'fetchT')
+    .mockImplementation(async () => (caddyOk
+      ? { ok: true, status: 200 }
+      : { ok: false, status: 403 }));
+
   const app = express();
   const config = {
     CONFIG_FILE: '/tmp/dc-test-config.json',
@@ -108,8 +117,10 @@ function buildApp({ configOk = true, servicesOk = true, dockerOk = true } = {})
       allOk = false;
     }
     try {
+      // DC-087 — mirror src/app.js exactly: fetchT (mocked above), not raw
+      // fetch. Hermetic: no live request to a real Caddy admin.
       const caddyUrl = config.CADDY_ADMIN_URL || 'http://localhost:2019';
-      const response = await fetch(`${caddyUrl}/config/apps/http/servers/srv0/listen`, { signal: AbortSignal.timeout(10000) });
+      const response = await fetchT(`${caddyUrl}/config/apps/http/servers/srv0/listen`, {}, 10000);
       checks.caddy = { ok: response.ok, status: response.status };
       if (!response.ok) allOk = false;
     } catch (e) {

dashcaddy-api/__tests__/utils-http-caddy-admin-origin.test.js

@@ -118,6 +118,67 @@ describe('Caddyfile + utils/http.js — Origin header construction (DC-051)', ()
     expect(offenders).toEqual([]);
   });
 
+  test('all :2019 call sites in TESTS use fetchT or a mocked fetchT (not raw fetch)', () => {
+    // DC-087 — the same rule, extended into __tests__. The api-code walk above
+    // skips __tests__, which let two mirrored health-handler test files keep a
+    // raw await-fetch caddy probe long after src/app.js moved to fetchT. On a
+    // host where the suite runs alongside a live Caddy admin (the prod box
+    // runs the full jest suite every 30 min via a cron adversarial check),
+    // that Origin-less raw fetch 403-spammed the Caddy journal (~700
+    // client-not-allowed error lines per day) while the tests still passed —
+    // checks.caddy.ok=false was silently accepted as sandbox noise. Mirrors
+    // MUST call fetchT (mocked at buildApp scope for hermeticity). A raw
+    // await-fetch at a Caddy-admin-URL call site in a test is an offender.
+    // NOTE: keep this comment free of backticks — stripComments pairs
+    // backtick spans across lines, and a stray pair shields real code from
+    // the comment stripper (this test self-flagged its first draft).
+    //
+    // Detection is deliberately FILE-LEVEL, not call-window: the historical
+    // drift kept the fetch call itself token-free (the URL came from a
+    // caddyUrl variable defined on a PREVIOUS line from CADDY_ADMIN_URL),
+    // so a call-window regex never fired. Any raw await-fetch in a file
+    // that also references the Caddy admin anywhere is an offender.
+    // Escape hatch for future tests that intentionally assert Origin-less
+    // 403 behavior against their own local listener: put the marker
+    // DC-087-ALLOW-RAW-FETCH in the file and it is skipped.
+    const testsRoot = path.join(__dirname);
+    const offenders = [];
+    const skipped = [];
+    function walk(dir) {
+      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (entry.name === 'node_modules') continue;
+        const p = path.join(dir, entry.name);
+        if (entry.isDirectory()) walk(p);
+        else if (entry.name.endsWith('.js')) {
+          const rawText = fs.readFileSync(p, 'utf8');
+          // Escape hatch (checked on RAW text so a comment marker works —
+          // comments are stripped below): a file carrying the
+          // DC-087-ALLOW-RAW-FETCH marker declares it intentionally
+          // raw-fetches the Caddy admin (e.g. asserting Origin-less 403
+          // against its own local listener). The guard file itself is
+          // always scanned (never skipped) so the hatch can't be used to
+          // blind this very test.
+          if (p !== __filename && /DC-087-ALLOW-RAW-FETCH/.test(rawText)) {
+            skipped.push(p);
+            continue;
+          }
+          const text = stripComments(rawText);
+          const hasAdminToken = /:2019|adminUrl|admin_api_url|CADDY_ADMIN/.test(text);
+          const hasRawAwaitFetch = /await\s+fetch\(/.test(text);
+          if (hasAdminToken && hasRawAwaitFetch) {
+            offenders.push(`${p}: raw await-fetch in a file referencing the Caddy admin (mock fetchT instead; documented escape-hatch marker available for intentional 403 tests)`);
+          }
+        }
+      }
+    }
+    walk(testsRoot);
+    if (skipped.length) {
+      // Visibility for hatch use — shows up in jest output for reviewers.
+      console.info('[DC-087 guard] escape-hatch skipped:', skipped.join(', '));
+    }
+    expect(offenders).toEqual([]);
+  });
+
   test('readiness handler in src/app.js probes the exact URL the watcher needs', () => {
     const raw = fs.readFileSync(
       path.join(__dirname, '../src/app.js'),
@@ -127,9 +188,9 @@ describe('Caddyfile + utils/http.js — Origin header construction (DC-051)', ()
     expect(raw).toMatch(/\/config\/apps\/http\/servers\/srv0\/listen/);
     // Goes through fetchT, NOT bare fetch — that's how the Origin injection
     // takes effect. Look at the 800 chars BEFORE the probe URL on the same
-    // line / call site — the call must be `fetchT(...)`, not `await fetch(...)`.
-    // (We look backward because the URL sits inside the call's argument list,
-    // so the call site comes before the URL token.)
+    // line / call site — the call must be fetchT(...), never a raw await of
+    // the global fetch. (We look backward because the URL sits inside the
+    // call's argument list, so the call site comes before the URL token.)
     const idx = raw.indexOf('srv0/listen');
     const around = raw.substr(Math.max(0, idx - 400), 800);
     expect(around).toMatch(/fetchT\(/);

dashcaddy-api/src/monitoring/health-checker.js

@@ -32,6 +32,10 @@ const CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_INTERVAL || '30000', 10
 const MAX_CHECK_INTERVAL = parseInt(process.env.HEALTH_CHECK_MAX_INTERVAL || '300000', 10); // 5 minutes max backoff
 const MAX_ENTRIES_PER_SERVICE = parseInt(process.env.HEALTH_MAX_ENTRIES || '500', 10); // Cap to prevent disk explosion
 const HISTORY_RETENTION_DAYS = parseInt(process.env.HEALTH_HISTORY_RETENTION || '30', 10);
+// DC-088: how long a removal tombstone outlives the removal itself. Only needs
+// to cover the max in-flight probe lifetime (timeout + scheduling headroom);
+// swept by cleanupHistory so removed services cannot accumulate map entries.
+const REMOVED_GENERATION_TTL_MS = parseInt(process.env.HEALTH_REMOVED_GEN_TTL || '600000', 10);
 
 // DC-086: hysteresis thresholds for badge display.
 // The raw probe result can flap on a single transient blip (Caddy reload,
@@ -72,6 +76,14 @@ class HealthChecker extends EventEmitter {
     this.serviceTimers = new Map(); // serviceId -> timer for per-service backoff
     // Invalidate probe completions that race with removal/reconfiguration.
     this.serviceGenerations = new Map(); // serviceId -> configuration generation
+    // DC-088: monotonically increasing sequence so generation numbers can never
+    // repeat across remove -> re-add cycles (prevents ABA on the stale check).
+    this.generationSeq = 0;
+    // DC-088: serviceId -> { generation, removedAt } tombstones. A live entry in
+    // serviceGenerations means the service is (re)configured; a tombstone with a
+    // HIGHER generation than the captured one marks the capture as stale. Entry
+    // is deleted when the service is removed, so the live map cannot leak.
+    this.removedGenerations = new Map();
   }
 
   /**
@@ -139,6 +151,21 @@ class HealthChecker extends EventEmitter {
     this.cleanupHistory();
   }
 
+  /**
+   * DC-088: true when a probe's captured generation no longer matches the
+   * service's current configuration state. A live serviceGenerations entry
+   * must match exactly. With no live entry the service was never configured
+   * in this process (disk-loaded / direct callers) — stale only if a removal
+   * tombstone with a HIGHER generation exists.
+   */
+  _isStaleCapture(serviceId, generation) {
+    if (this.serviceGenerations.has(serviceId)) {
+      return this.serviceGenerations.get(serviceId) !== generation;
+    }
+    const tomb = this.removedGenerations.get(serviceId);
+    return Boolean(tomb && tomb.generation > generation);
+  }
+
   /**
    * Check a single service
    */
@@ -160,7 +187,7 @@ class HealthChecker extends EventEmitter {
         details: result.details
       };
 
-      if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
+      if (this._isStaleCapture(serviceId, generation)) {
         return status;
       }
 
@@ -179,9 +206,6 @@ class HealthChecker extends EventEmitter {
     } catch (error) {
       const responseTime = Date.now() - startTime;
 
-      // Increment failure count for backoff
-      this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
-
       const status = {
         serviceId,
         timestamp: new Date().toISOString(),
@@ -190,10 +214,14 @@ class HealthChecker extends EventEmitter {
         error: error.message
       };
 
-      if ((this.serviceGenerations.get(serviceId) || 0) !== generation) {
+      if (this._isStaleCapture(serviceId, generation)) {
         return status;
       }
 
+      // Increment failure count for backoff — only after the result is known
+      // to be non-stale, so a removed service cannot re-create map entries.
+      this.consecutiveFailures.set(serviceId, (this.consecutiveFailures.get(serviceId) || 0) + 1);
+
       const previousStatus = this.currentStatus.get(serviceId);
       this.recordStatus(serviceId, status);
       this.checkForIncidents(serviceId, status, config, previousStatus);
@@ -660,7 +688,13 @@ class HealthChecker extends EventEmitter {
       this.config.services = {};
     }
 
-    this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
+    // DC-088: monotonic instance-wide sequence — a re-added service can never
+    // recycle a previous generation number, and any older in-flight capture is
+    // invalidated by definition.
+    this.generationSeq += 1;
+    this.serviceGenerations.set(serviceId, this.generationSeq);
+    // Re-configuration supersedes any prior removal tombstone.
+    this.removedGenerations.delete(serviceId);
     this.config.services[serviceId] = {
       enabled: config.enabled !== false,
       name: config.name || serviceId,
@@ -683,12 +717,35 @@ class HealthChecker extends EventEmitter {
    * Remove service configuration
    */
   removeService(serviceId) {
-    this.serviceGenerations.set(serviceId, (this.serviceGenerations.get(serviceId) || 0) + 1);
+    // DC-088: tombstone the captured generation instead of leaking an entry.
+    // The live map entry is deleted; an in-flight probe captured BEFORE this
+    // point sees no live entry but a higher tombstone generation, so it is
+    // discarded. configureService clears the tombstone on re-add.
+    this.generationSeq += 1;
+    this.serviceGenerations.delete(serviceId);
+    this.removedGenerations.set(serviceId, {
+      generation: this.generationSeq,
+      removedAt: Date.now()
+    });
     if (this.config.services) {
       delete this.config.services[serviceId];
       this.saveConfig();
     }
 
+    // DC-088: open incidents for a removed service must not linger forever.
+    // Close them through the same resolve path a recovery would, annotated so
+    // history shows why (dashboard renders resolved incidents green + duration).
+    for (const incident of this.incidents) {
+      if (incident.serviceId === serviceId && incident.status === 'open') {
+        incident.status = 'resolved';
+        incident.resolvedAt = new Date().toISOString();
+        incident.duration = new Date(incident.resolvedAt) - new Date(incident.createdAt);
+        incident.resolvedBy = 'service-removed';
+        this.emit('incident-resolved', incident);
+        this.emit('log', 'info', `Incident closed by service removal: ${incident.id}`);
+      }
+    }
+
     this.currentStatus.delete(serviceId);
     this.displayedStatus.delete(serviceId);
     this.consecutiveSinceChange.delete(serviceId);
@@ -714,6 +771,18 @@ class HealthChecker extends EventEmitter {
         this.history[serviceId] = this.history[serviceId].slice(-MAX_ENTRIES_PER_SERVICE);
       }
     }
+
+    // DC-088: sweep expired removal tombstones. After the TTL no probe that
+    // captured a pre-removal generation can still be in flight (timeout is
+    // bounded by performHealthCheck), so the tombstone has done its job.
+    if (this.removedGenerations.size > 0) {
+      const now = Date.now();
+      for (const [serviceId, tomb] of this.removedGenerations) {
+        if (now - tomb.removedAt > REMOVED_GENERATION_TTL_MS) {
+          this.removedGenerations.delete(serviceId);
+        }
+      }
+    }
   }
 
   /**

status/sw.js

@@ -1,4 +1,4 @@
-const CACHE = 'dashcaddy-shell-c25cea8485';
+const CACHE = 'dashcaddy-shell-497e1f671c';
 const PRECACHE = [
   '/',
   '/index.html',