DC-067: graceful shutdown coordinator (EventEmitter, 10s drain, idempotent)

dashcaddy-api/__tests__/shutdown-coordinator.test.js

@@ -0,0 +1,318 @@
+/**
+ * Tests for the graceful shutdown coordinator (DC-067).
+ *
+ * Covers:
+ *  - Constructor rejects bad inputs
+ *  - shutdown() emits 'shutdown' event with the signal name
+ *  - shutdown() stops each manager in declaration order
+ *  - shutdown() is idempotent — second call logs and returns
+ *  - shutdown() force-exits after drainTimeoutMs if server.close never fires
+ *  - shutdown() clears the force-exit timer when server.close fires first
+ *  - shutdown() catches manager.stop() throws so one bad manager doesn't
+ *    prevent the others from being stopped
+ *  - installSignalHandlers() registers for SIGTERM and SIGINT by default
+ *
+ * process.exit is mocked so tests don't actually kill the test runner.
+ */
+'use strict';
+
+const EventEmitter = require('events');
+const {
+  createShutdownCoordinator,
+  installSignalHandlers,
+  DEFAULT_DRAIN_TIMEOUT_MS,
+  ShutdownCoordinator,
+} = require('../src/utilities/shutdown');
+
+describe('ShutdownCoordinator (DC-067)', () => {
+  let exitMock;
+  let exitCalls;
+
+  beforeEach(() => {
+    exitCalls = [];
+    // Use jest.spyOn so the mock is restored in afterEach (via clearMocks +
+    // restoreMocks: true in jest.config.js). Direct assignment to process.exit
+    // doesn't suppress Jest's process.exit watchlist which fails the test.
+    exitMock = jest.spyOn(process, 'exit').mockImplementation((code) => {
+      exitCalls.push(code);
+      // Returning undefined prevents the test runner from actually exiting.
+      return undefined;
+    });
+  });
+
+  afterEach(() => {
+    exitMock.mockRestore();
+    jest.clearAllTimers();
+  });
+
+  function makeFakeServer() {
+    // Synchronous close callback. async (setImmediate) callbacks would fire
+    // AFTER the test that triggered shutdown has ended, hitting the real
+    // process.exit in the restored mock window.
+    return { close: jest.fn((cb) => { cb(); }) };
+  }
+
+  function makeFakeLog() {
+    return {
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+    };
+  }
+
+  describe('constructor', () => {
+    test('throws if server is missing', () => {
+      expect(() => createShutdownCoordinator({ log: makeFakeLog(), managers: [] }))
+        .toThrow('server is required');
+    });
+
+    test('throws if log is missing or invalid', () => {
+      expect(() => createShutdownCoordinator({ server: makeFakeServer(), managers: [] }))
+        .toThrow('log with info/warn is required');
+      expect(() => createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: { foo: 'bar' },
+        managers: [],
+      })).toThrow('log with info/warn is required');
+    });
+
+    test('uses DEFAULT_DRAIN_TIMEOUT_MS when drainTimeoutMs is not finite positive', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        drainTimeoutMs: 0,
+        managers: [],
+      });
+      expect(c.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
+
+      const c2 = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        drainTimeoutMs: NaN,
+        managers: [],
+      });
+      expect(c2.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
+
+      const c3 = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        drainTimeoutMs: 5000,
+        managers: [],
+      });
+      expect(c3.drainTimeoutMs).toBe(5000);
+    });
+
+    test('defaults managers to [] when not an array', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+      });
+      expect(c.managers).toEqual([]);
+    });
+
+    test('is an EventEmitter', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers: [],
+      });
+      expect(c).toBeInstanceOf(EventEmitter);
+      expect(c).toBeInstanceOf(ShutdownCoordinator);
+    });
+  });
+
+  describe('shutdown()', () => {
+    test('emits shutdown event with signal name', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers: [],
+      });
+      const handler = jest.fn();
+      c.on('shutdown', handler);
+
+      c.shutdown('SIGTERM');
+
+      expect(handler).toHaveBeenCalledTimes(1);
+      expect(handler).toHaveBeenCalledWith('SIGTERM');
+    });
+
+    test('calls server.close() once', () => {
+      const server = makeFakeServer();
+      const c = createShutdownCoordinator({
+        server,
+        log: makeFakeLog(),
+        managers: [],
+      });
+
+      c.shutdown('SIGTERM');
+
+      expect(server.close).toHaveBeenCalledTimes(1);
+    });
+
+    test('stops each manager in declaration order', async () => {
+      const order = [];
+      const managers = [
+        { name: 'first', stop: jest.fn(() => { order.push('first'); }) },
+        { name: 'second', stop: jest.fn(() => { order.push('second'); }) },
+        { name: 'third', stop: jest.fn(() => { order.push('third'); }) },
+      ];
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers,
+      });
+
+      c.shutdown('SIGTERM');
+
+      // Wait one microtask + macrotask for the async chain to settle.
+      await new Promise((resolve) => setImmediate(resolve));
+
+      expect(order).toEqual(['first', 'second', 'third']);
+    });
+
+    test('continues stopping remaining managers if one throws', async () => {
+      const order = [];
+      const managers = [
+        { name: 'first', stop: jest.fn(() => { order.push('first'); }) },
+        { name: 'broken', stop: jest.fn(() => { throw new Error('boom'); }) },
+        { name: 'third', stop: jest.fn(() => { order.push('third'); }) },
+      ];
+      const log = makeFakeLog();
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log,
+        managers,
+      });
+
+      c.shutdown('SIGTERM');
+      await new Promise((resolve) => setImmediate(resolve));
+
+      expect(order).toEqual(['first', 'third']);
+      expect(log.warn).toHaveBeenCalledWith(
+        'shutdown',
+        'manager stop failed: broken',
+        expect.objectContaining({ error: 'boom' }),
+      );
+    });
+
+    test('is idempotent — second shutdown() returns without re-running', () => {
+      const server = makeFakeServer();
+      const c = createShutdownCoordinator({
+        server,
+        log: makeFakeLog(),
+        managers: [{ name: 'm', stop: jest.fn() }],
+      });
+
+      c.shutdown('SIGTERM');
+      c.shutdown('SIGTERM');
+      c.shutdown('SIGINT');
+
+      expect(server.close).toHaveBeenCalledTimes(1);
+      expect(c.isShuttingDown()).toBe(true);
+    });
+
+    test('isShuttingDown() flips false→true on first shutdown call', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers: [],
+      });
+      expect(c.isShuttingDown()).toBe(false);
+      c.shutdown('SIGTERM');
+      expect(c.isShuttingDown()).toBe(true);
+    });
+
+    test('force-exits after drainTimeoutMs if server.close never fires', () => {
+      jest.useFakeTimers();
+      try {
+        const server = { close: jest.fn(/* never calls back */) };
+        const log = makeFakeLog();
+        const c = createShutdownCoordinator({
+          server,
+          log,
+          drainTimeoutMs: 1000,
+          managers: [],
+        });
+
+        c.shutdown('SIGTERM');
+        expect(exitCalls).toEqual([]);
+
+        jest.advanceTimersByTime(999);
+        expect(exitCalls).toEqual([]);
+
+        jest.advanceTimersByTime(2);
+        expect(exitCalls).toEqual([0]);
+        expect(log.warn).toHaveBeenCalledWith(
+          'shutdown',
+          'drain timeout (1000ms) reached, force-exiting',
+        );
+      } finally {
+        jest.useRealTimers();
+      }
+    });
+
+    test('clears force-exit timer when server.close fires first', () => {
+      jest.useFakeTimers();
+      try {
+        const server = makeFakeServer();  // calls back on setImmediate
+        const log = makeFakeLog();
+        const c = createShutdownCoordinator({
+          server,
+          log,
+          drainTimeoutMs: 1000,
+          managers: [],
+        });
+
+        c.shutdown('SIGTERM');
+        // Flush the setImmediate so server.close callback fires.
+        jest.runOnlyPendingTimers();
+        jest.advanceTimersByTime(0);
+
+        expect(exitCalls).toEqual([0]);
+
+        // Even after the full drain timeout, no extra exit should fire
+        // (because we cleared the timer in the close callback).
+        jest.advanceTimersByTime(5000);
+        expect(exitCalls).toEqual([0]);
+      } finally {
+        jest.useRealTimers();
+      }
+    });
+  });
+
+  describe('installSignalHandlers()', () => {
+    test('registers listeners on the given signals', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers: [],
+      });
+      const shutdownSpy = jest.spyOn(c, 'shutdown');
+
+      installSignalHandlers(c);
+
+      // Emit fake signals through process.emit to verify the listener was
+      // registered (process.on listens to the process EventEmitter).
+      process.emit('SIGTERM');
+      process.emit('SIGINT');
+
+      expect(shutdownSpy).toHaveBeenCalledWith('SIGTERM');
+      expect(shutdownSpy).toHaveBeenCalledWith('SIGINT');
+    });
+
+    test('accepts custom signal list', () => {
+      const c = createShutdownCoordinator({
+        server: makeFakeServer(),
+        log: makeFakeLog(),
+        managers: [],
+      });
+      const shutdownSpy = jest.spyOn(c, 'shutdown');
+
+      installSignalHandlers(c, ['SIGHUP']);
+
+      process.emit('SIGHUP');
+      expect(shutdownSpy).toHaveBeenCalledWith('SIGHUP');
+    });
+  });
+});

dashcaddy-api/server.js

@@ -252,43 +252,52 @@ process.on('uncaughtException', (error) => {
       log.info('server', 'All feature modules initialized');
     });
 
-    // Graceful shutdown
+    // Graceful shutdown (DC-067) — drains in-flight HTTP connections, stops
-    const shutdown = (signal) => {
+    // each manager in deterministic order, emits a 'shutdown' event for any
-      log.info('shutdown', `${signal} received, draining connections...`);
+    // additional listeners, and force-exits after a 10s drain timeout.
-      
+    // Idempotent: a second SIGTERM during shutdown is a no-op.
-      const resourceMonitor = require('./src/managers/resource-monitor');
+    const {
-      const backupManager = require('./src/utilities/backup-manager');
+      createShutdownCoordinator,
-      const healthChecker = require('./src/monitoring/health-checker');
+      installSignalHandlers,
-      const updateManager = require('./src/managers/update-manager');
+      DEFAULT_DRAIN_TIMEOUT_MS,
-      const selfUpdater = require('./src/docker/self-updater');
+    } = require('./src/utilities/shutdown');
-      
-      resourceMonitor.stop();
-      backupManager.stop();
-      healthChecker.stop();
-      updateManager.stop();
-      selfUpdater.stop();
 
+    const optionalManagers = [];
     try {
-        const dockerMaintenance = require('./src/docker/docker-maintenance');
+      optionalManagers.push({
-        dockerMaintenance.stop();
+        name: 'docker-maintenance',
-      } catch { /* optional */ }
+        stop: () => require('./src/docker/docker-maintenance').stop(),
-      
+      });
+    } catch { /* optional module */ }
     try {
-        const logDigest = require('./src/security/log-digest');
+      optionalManagers.push({
-        logDigest.stop();
+        name: 'log-digest',
-      } catch { /* optional */ }
+        stop: () => require('./src/security/log-digest').stop(),
+      });
+    } catch { /* optional module */ }
 
-      server.close(() => {
+    const coordinator = createShutdownCoordinator({
-        log.info('shutdown', 'HTTP server closed');
+      server,
-        process.exit(0);
+      log,
+      drainTimeoutMs: DEFAULT_DRAIN_TIMEOUT_MS,
+      managers: [
+        { name: 'resource-monitor', stop: () => require('./src/managers/resource-monitor').stop() },
+        { name: 'backup-manager', stop: () => require('./src/utilities/backup-manager').stop() },
+        { name: 'health-checker', stop: () => require('./src/monitoring/health-checker').stop() },
+        { name: 'update-manager', stop: () => require('./src/managers/update-manager').stop() },
+        { name: 'self-updater', stop: () => require('./src/docker/self-updater').stop() },
+        ...optionalManagers,
+      ],
     });
 
-      // Force exit after 5s if connections don't drain
+    // Expose the shutdown signal as an event so additional listeners can
-      setTimeout(() => process.exit(0), 5000).unref();
+    // subscribe without touching this file. The coordinator is an
-    };
+    // EventEmitter and emits 'shutdown' on SIGTERM/SIGINT.
+    coordinator.on('shutdown', (signal) => {
+      log.info('shutdown', 'shutdown event observed', { signal });
+    });
 
-    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    installSignalHandlers(coordinator);
-    process.on('SIGINT', () => shutdown('SIGINT'));
 
   } catch (error) {
     console.error('[FATAL] Server startup failed:', error);

dashcaddy-api/src/utilities/shutdown.js

@@ -0,0 +1,133 @@
+/**
+ * Graceful shutdown coordinator — DashCaddy
+ *
+ * Extracts the SIGTERM/SIGINT handler from server.js into a testable,
+ * reusable module that:
+ *   1. Calls server.close() to drain in-flight HTTP connections
+ *   2. Stops each manager in a deterministic order
+ *   3. Emits a 'shutdown' event so additional listeners can do cleanup
+ *   4. Force-exits after a configurable drain timeout if connections don't drain
+ *   5. Is idempotent — a second SIGTERM during shutdown does not re-run handlers
+ *
+ * Spec: DC-067 (production-grade backlog). Docker sends SIGTERM on stop;
+ * without this coordinator, in-flight API calls drop.
+ *
+ * Exports:
+ *   - createShutdownCoordinator({ server, log, drainTimeoutMs, managers })
+ *       Returns an EventEmitter with: { shutdown, isShuttingDown, on, emit, ... }
+ *   - installSignalHandlers(coordinator, signals = ['SIGTERM', 'SIGINT'])
+ *       Registers the OS-level handlers. Idempotent.
+ */
+'use strict';
+
+const EventEmitter = require('events');
+
+const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
+
+class ShutdownCoordinator extends EventEmitter {
+  constructor({ server, log, drainTimeoutMs, managers }) {
+    super();
+    if (!server) throw new Error('createShutdownCoordinator: server is required');
+    if (!log || typeof log.info !== 'function') {
+      throw new Error('createShutdownCoordinator: log with info/warn is required');
+    }
+    this.server = server;
+    this.log = log;
+    this.drainTimeoutMs = Number.isFinite(drainTimeoutMs) && drainTimeoutMs > 0
+      ? drainTimeoutMs
+      : DEFAULT_DRAIN_TIMEOUT_MS;
+    this.managers = Array.isArray(managers) ? managers : [];
+    this._shuttingDown = false;
+    this._forceTimer = null;
+  }
+
+  isShuttingDown() {
+    return this._shuttingDown;
+  }
+
+  async _stopManager(m) {
+    try {
+      await m.stop();
+      this.log.info('shutdown', `manager stopped: ${m.name}`);
+    } catch (err) {
+      this.log.warn('shutdown', `manager stop failed: ${m.name}`, { error: err.message });
+    }
+  }
+
+  shutdown(signal) {
+    if (this._shuttingDown) {
+      this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
+      return;
+    }
+    this._shuttingDown = true;
+    this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
+
+    // Emit 'shutdown' event first so any listeners can flush their state
+    // before the managers start stopping.
+    this.emit('shutdown', signal);
+
+    // Stop each manager in order. Fire-and-forget — each manager's stop()
+    // is expected to be fast (cancel timers, flush buffers). If a manager
+    // has long async work, it should expose its own drain mechanism.
+    for (const m of this.managers) {
+      this._stopManager(m);
+    }
+
+    // Close the HTTP server. Stops accepting new connections, waits for
+    // in-flight requests to complete naturally.
+    let closed = false;
+    try {
+      this.server.close(() => {
+        closed = true;
+        this.log.info('shutdown', 'HTTP server closed cleanly');
+        if (this._forceTimer) {
+          clearTimeout(this._forceTimer);
+          this._forceTimer = null;
+        }
+        this.emit('closed', signal);
+        process.exit(0);
+      });
+    } catch (err) {
+      this.log.error('shutdown', 'server.close threw', { error: err.message });
+    }
+
+    // Force-exit if drain doesn't complete in time.
+    this._forceTimer = setTimeout(() => {
+      if (closed) return;  // race: server.close fired AND timer fired
+      this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`);
+      process.exit(0);
+    }, this.drainTimeoutMs);
+    if (this._forceTimer && typeof this._forceTimer.unref === 'function') {
+      this._forceTimer.unref();
+    }
+  }
+}
+
+function createShutdownCoordinator(opts) {
+  return new ShutdownCoordinator(opts);
+}
+
+/**
+ * Install OS-level signal handlers. Idempotent: second call is a no-op.
+ *
+ * @param {ShutdownCoordinator} coordinator
+ * @param {string[]} signals - signals to handle (default: SIGTERM, SIGINT)
+ */
+function installSignalHandlers(coordinator, signals) {
+  if (!coordinator || typeof coordinator.shutdown !== 'function') {
+    throw new Error('installSignalHandlers: coordinator required');
+  }
+  const sigs = Array.isArray(signals) && signals.length > 0
+    ? signals
+    : ['SIGTERM', 'SIGINT'];
+  for (const sig of sigs) {
+    process.on(sig, () => coordinator.shutdown(sig));
+  }
+}
+
+module.exports = {
+  createShutdownCoordinator,
+  installSignalHandlers,
+  DEFAULT_DRAIN_TIMEOUT_MS,
+  ShutdownCoordinator,  // exported for tests
+};