From 88ff260e5e6826aef6e731e45a88a9b6796a3c2b Mon Sep 17 00:00:00 2001 From: hermes Date: Wed, 12 Aug 2026 03:36:49 -0700 Subject: [PATCH] DC-067: graceful shutdown coordinator (EventEmitter, 10s drain, idempotent) --- .../__tests__/shutdown-coordinator.test.js | 318 ++++++++++++++++++ dashcaddy-api/server.js | 77 +++-- dashcaddy-api/src/utilities/shutdown.js | 133 ++++++++ 3 files changed, 494 insertions(+), 34 deletions(-) create mode 100644 dashcaddy-api/__tests__/shutdown-coordinator.test.js create mode 100644 dashcaddy-api/src/utilities/shutdown.js diff --git a/dashcaddy-api/__tests__/shutdown-coordinator.test.js b/dashcaddy-api/__tests__/shutdown-coordinator.test.js new file mode 100644 index 0000000..456db5f --- /dev/null +++ b/dashcaddy-api/__tests__/shutdown-coordinator.test.js @@ -0,0 +1,318 @@ +/** + * Tests for the graceful shutdown coordinator (DC-067). + * + * Covers: + * - Constructor rejects bad inputs + * - shutdown() emits 'shutdown' event with the signal name + * - shutdown() stops each manager in declaration order + * - shutdown() is idempotent — second call logs and returns + * - shutdown() force-exits after drainTimeoutMs if server.close never fires + * - shutdown() clears the force-exit timer when server.close fires first + * - shutdown() catches manager.stop() throws so one bad manager doesn't + * prevent the others from being stopped + * - installSignalHandlers() registers for SIGTERM and SIGINT by default + * + * process.exit is mocked so tests don't actually kill the test runner. + */ +'use strict'; + +const EventEmitter = require('events'); +const { + createShutdownCoordinator, + installSignalHandlers, + DEFAULT_DRAIN_TIMEOUT_MS, + ShutdownCoordinator, +} = require('../src/utilities/shutdown'); + +describe('ShutdownCoordinator (DC-067)', () => { + let exitMock; + let exitCalls; + + beforeEach(() => { + exitCalls = []; + // Use jest.spyOn so the mock is restored in afterEach (via clearMocks + + // restoreMocks: true in jest.config.js). Direct assignment to process.exit + // doesn't suppress Jest's process.exit watchlist which fails the test. + exitMock = jest.spyOn(process, 'exit').mockImplementation((code) => { + exitCalls.push(code); + // Returning undefined prevents the test runner from actually exiting. + return undefined; + }); + }); + + afterEach(() => { + exitMock.mockRestore(); + jest.clearAllTimers(); + }); + + function makeFakeServer() { + // Synchronous close callback. async (setImmediate) callbacks would fire + // AFTER the test that triggered shutdown has ended, hitting the real + // process.exit in the restored mock window. + return { close: jest.fn((cb) => { cb(); }) }; + } + + function makeFakeLog() { + return { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }; + } + + describe('constructor', () => { + test('throws if server is missing', () => { + expect(() => createShutdownCoordinator({ log: makeFakeLog(), managers: [] })) + .toThrow('server is required'); + }); + + test('throws if log is missing or invalid', () => { + expect(() => createShutdownCoordinator({ server: makeFakeServer(), managers: [] })) + .toThrow('log with info/warn is required'); + expect(() => createShutdownCoordinator({ + server: makeFakeServer(), + log: { foo: 'bar' }, + managers: [], + })).toThrow('log with info/warn is required'); + }); + + test('uses DEFAULT_DRAIN_TIMEOUT_MS when drainTimeoutMs is not finite positive', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + drainTimeoutMs: 0, + managers: [], + }); + expect(c.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS); + + const c2 = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + drainTimeoutMs: NaN, + managers: [], + }); + expect(c2.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS); + + const c3 = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + drainTimeoutMs: 5000, + managers: [], + }); + expect(c3.drainTimeoutMs).toBe(5000); + }); + + test('defaults managers to [] when not an array', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + }); + expect(c.managers).toEqual([]); + }); + + test('is an EventEmitter', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers: [], + }); + expect(c).toBeInstanceOf(EventEmitter); + expect(c).toBeInstanceOf(ShutdownCoordinator); + }); + }); + + describe('shutdown()', () => { + test('emits shutdown event with signal name', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers: [], + }); + const handler = jest.fn(); + c.on('shutdown', handler); + + c.shutdown('SIGTERM'); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith('SIGTERM'); + }); + + test('calls server.close() once', () => { + const server = makeFakeServer(); + const c = createShutdownCoordinator({ + server, + log: makeFakeLog(), + managers: [], + }); + + c.shutdown('SIGTERM'); + + expect(server.close).toHaveBeenCalledTimes(1); + }); + + test('stops each manager in declaration order', async () => { + const order = []; + const managers = [ + { name: 'first', stop: jest.fn(() => { order.push('first'); }) }, + { name: 'second', stop: jest.fn(() => { order.push('second'); }) }, + { name: 'third', stop: jest.fn(() => { order.push('third'); }) }, + ]; + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers, + }); + + c.shutdown('SIGTERM'); + + // Wait one microtask + macrotask for the async chain to settle. + await new Promise((resolve) => setImmediate(resolve)); + + expect(order).toEqual(['first', 'second', 'third']); + }); + + test('continues stopping remaining managers if one throws', async () => { + const order = []; + const managers = [ + { name: 'first', stop: jest.fn(() => { order.push('first'); }) }, + { name: 'broken', stop: jest.fn(() => { throw new Error('boom'); }) }, + { name: 'third', stop: jest.fn(() => { order.push('third'); }) }, + ]; + const log = makeFakeLog(); + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log, + managers, + }); + + c.shutdown('SIGTERM'); + await new Promise((resolve) => setImmediate(resolve)); + + expect(order).toEqual(['first', 'third']); + expect(log.warn).toHaveBeenCalledWith( + 'shutdown', + 'manager stop failed: broken', + expect.objectContaining({ error: 'boom' }), + ); + }); + + test('is idempotent — second shutdown() returns without re-running', () => { + const server = makeFakeServer(); + const c = createShutdownCoordinator({ + server, + log: makeFakeLog(), + managers: [{ name: 'm', stop: jest.fn() }], + }); + + c.shutdown('SIGTERM'); + c.shutdown('SIGTERM'); + c.shutdown('SIGINT'); + + expect(server.close).toHaveBeenCalledTimes(1); + expect(c.isShuttingDown()).toBe(true); + }); + + test('isShuttingDown() flips false→true on first shutdown call', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers: [], + }); + expect(c.isShuttingDown()).toBe(false); + c.shutdown('SIGTERM'); + expect(c.isShuttingDown()).toBe(true); + }); + + test('force-exits after drainTimeoutMs if server.close never fires', () => { + jest.useFakeTimers(); + try { + const server = { close: jest.fn(/* never calls back */) }; + const log = makeFakeLog(); + const c = createShutdownCoordinator({ + server, + log, + drainTimeoutMs: 1000, + managers: [], + }); + + c.shutdown('SIGTERM'); + expect(exitCalls).toEqual([]); + + jest.advanceTimersByTime(999); + expect(exitCalls).toEqual([]); + + jest.advanceTimersByTime(2); + expect(exitCalls).toEqual([0]); + expect(log.warn).toHaveBeenCalledWith( + 'shutdown', + 'drain timeout (1000ms) reached, force-exiting', + ); + } finally { + jest.useRealTimers(); + } + }); + + test('clears force-exit timer when server.close fires first', () => { + jest.useFakeTimers(); + try { + const server = makeFakeServer(); // calls back on setImmediate + const log = makeFakeLog(); + const c = createShutdownCoordinator({ + server, + log, + drainTimeoutMs: 1000, + managers: [], + }); + + c.shutdown('SIGTERM'); + // Flush the setImmediate so server.close callback fires. + jest.runOnlyPendingTimers(); + jest.advanceTimersByTime(0); + + expect(exitCalls).toEqual([0]); + + // Even after the full drain timeout, no extra exit should fire + // (because we cleared the timer in the close callback). + jest.advanceTimersByTime(5000); + expect(exitCalls).toEqual([0]); + } finally { + jest.useRealTimers(); + } + }); + }); + + describe('installSignalHandlers()', () => { + test('registers listeners on the given signals', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers: [], + }); + const shutdownSpy = jest.spyOn(c, 'shutdown'); + + installSignalHandlers(c); + + // Emit fake signals through process.emit to verify the listener was + // registered (process.on listens to the process EventEmitter). + process.emit('SIGTERM'); + process.emit('SIGINT'); + + expect(shutdownSpy).toHaveBeenCalledWith('SIGTERM'); + expect(shutdownSpy).toHaveBeenCalledWith('SIGINT'); + }); + + test('accepts custom signal list', () => { + const c = createShutdownCoordinator({ + server: makeFakeServer(), + log: makeFakeLog(), + managers: [], + }); + const shutdownSpy = jest.spyOn(c, 'shutdown'); + + installSignalHandlers(c, ['SIGHUP']); + + process.emit('SIGHUP'); + expect(shutdownSpy).toHaveBeenCalledWith('SIGHUP'); + }); + }); +}); diff --git a/dashcaddy-api/server.js b/dashcaddy-api/server.js index 3c70106..50b2332 100644 --- a/dashcaddy-api/server.js +++ b/dashcaddy-api/server.js @@ -252,43 +252,52 @@ process.on('uncaughtException', (error) => { log.info('server', 'All feature modules initialized'); }); - // Graceful shutdown - const shutdown = (signal) => { - log.info('shutdown', `${signal} received, draining connections...`); - - const resourceMonitor = require('./src/managers/resource-monitor'); - const backupManager = require('./src/utilities/backup-manager'); - const healthChecker = require('./src/monitoring/health-checker'); - const updateManager = require('./src/managers/update-manager'); - const selfUpdater = require('./src/docker/self-updater'); - - resourceMonitor.stop(); - backupManager.stop(); - healthChecker.stop(); - updateManager.stop(); - selfUpdater.stop(); - - try { - const dockerMaintenance = require('./src/docker/docker-maintenance'); - dockerMaintenance.stop(); - } catch { /* optional */ } - - try { - const logDigest = require('./src/security/log-digest'); - logDigest.stop(); - } catch { /* optional */ } + // Graceful shutdown (DC-067) — drains in-flight HTTP connections, stops + // each manager in deterministic order, emits a 'shutdown' event for any + // additional listeners, and force-exits after a 10s drain timeout. + // Idempotent: a second SIGTERM during shutdown is a no-op. + const { + createShutdownCoordinator, + installSignalHandlers, + DEFAULT_DRAIN_TIMEOUT_MS, + } = require('./src/utilities/shutdown'); - server.close(() => { - log.info('shutdown', 'HTTP server closed'); - process.exit(0); + const optionalManagers = []; + try { + optionalManagers.push({ + name: 'docker-maintenance', + stop: () => require('./src/docker/docker-maintenance').stop(), }); - - // Force exit after 5s if connections don't drain - setTimeout(() => process.exit(0), 5000).unref(); - }; + } catch { /* optional module */ } + try { + optionalManagers.push({ + name: 'log-digest', + stop: () => require('./src/security/log-digest').stop(), + }); + } catch { /* optional module */ } - process.on('SIGTERM', () => shutdown('SIGTERM')); - process.on('SIGINT', () => shutdown('SIGINT')); + const coordinator = createShutdownCoordinator({ + server, + log, + drainTimeoutMs: DEFAULT_DRAIN_TIMEOUT_MS, + managers: [ + { name: 'resource-monitor', stop: () => require('./src/managers/resource-monitor').stop() }, + { name: 'backup-manager', stop: () => require('./src/utilities/backup-manager').stop() }, + { name: 'health-checker', stop: () => require('./src/monitoring/health-checker').stop() }, + { name: 'update-manager', stop: () => require('./src/managers/update-manager').stop() }, + { name: 'self-updater', stop: () => require('./src/docker/self-updater').stop() }, + ...optionalManagers, + ], + }); + + // Expose the shutdown signal as an event so additional listeners can + // subscribe without touching this file. The coordinator is an + // EventEmitter and emits 'shutdown' on SIGTERM/SIGINT. + coordinator.on('shutdown', (signal) => { + log.info('shutdown', 'shutdown event observed', { signal }); + }); + + installSignalHandlers(coordinator); } catch (error) { console.error('[FATAL] Server startup failed:', error); diff --git a/dashcaddy-api/src/utilities/shutdown.js b/dashcaddy-api/src/utilities/shutdown.js new file mode 100644 index 0000000..a7db89d --- /dev/null +++ b/dashcaddy-api/src/utilities/shutdown.js @@ -0,0 +1,133 @@ +/** + * Graceful shutdown coordinator — DashCaddy + * + * Extracts the SIGTERM/SIGINT handler from server.js into a testable, + * reusable module that: + * 1. Calls server.close() to drain in-flight HTTP connections + * 2. Stops each manager in a deterministic order + * 3. Emits a 'shutdown' event so additional listeners can do cleanup + * 4. Force-exits after a configurable drain timeout if connections don't drain + * 5. Is idempotent — a second SIGTERM during shutdown does not re-run handlers + * + * Spec: DC-067 (production-grade backlog). Docker sends SIGTERM on stop; + * without this coordinator, in-flight API calls drop. + * + * Exports: + * - createShutdownCoordinator({ server, log, drainTimeoutMs, managers }) + * Returns an EventEmitter with: { shutdown, isShuttingDown, on, emit, ... } + * - installSignalHandlers(coordinator, signals = ['SIGTERM', 'SIGINT']) + * Registers the OS-level handlers. Idempotent. + */ +'use strict'; + +const EventEmitter = require('events'); + +const DEFAULT_DRAIN_TIMEOUT_MS = 10_000; + +class ShutdownCoordinator extends EventEmitter { + constructor({ server, log, drainTimeoutMs, managers }) { + super(); + if (!server) throw new Error('createShutdownCoordinator: server is required'); + if (!log || typeof log.info !== 'function') { + throw new Error('createShutdownCoordinator: log with info/warn is required'); + } + this.server = server; + this.log = log; + this.drainTimeoutMs = Number.isFinite(drainTimeoutMs) && drainTimeoutMs > 0 + ? drainTimeoutMs + : DEFAULT_DRAIN_TIMEOUT_MS; + this.managers = Array.isArray(managers) ? managers : []; + this._shuttingDown = false; + this._forceTimer = null; + } + + isShuttingDown() { + return this._shuttingDown; + } + + async _stopManager(m) { + try { + await m.stop(); + this.log.info('shutdown', `manager stopped: ${m.name}`); + } catch (err) { + this.log.warn('shutdown', `manager stop failed: ${m.name}`, { error: err.message }); + } + } + + shutdown(signal) { + if (this._shuttingDown) { + this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`); + return; + } + this._shuttingDown = true; + this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`); + + // Emit 'shutdown' event first so any listeners can flush their state + // before the managers start stopping. + this.emit('shutdown', signal); + + // Stop each manager in order. Fire-and-forget — each manager's stop() + // is expected to be fast (cancel timers, flush buffers). If a manager + // has long async work, it should expose its own drain mechanism. + for (const m of this.managers) { + this._stopManager(m); + } + + // Close the HTTP server. Stops accepting new connections, waits for + // in-flight requests to complete naturally. + let closed = false; + try { + this.server.close(() => { + closed = true; + this.log.info('shutdown', 'HTTP server closed cleanly'); + if (this._forceTimer) { + clearTimeout(this._forceTimer); + this._forceTimer = null; + } + this.emit('closed', signal); + process.exit(0); + }); + } catch (err) { + this.log.error('shutdown', 'server.close threw', { error: err.message }); + } + + // Force-exit if drain doesn't complete in time. + this._forceTimer = setTimeout(() => { + if (closed) return; // race: server.close fired AND timer fired + this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`); + process.exit(0); + }, this.drainTimeoutMs); + if (this._forceTimer && typeof this._forceTimer.unref === 'function') { + this._forceTimer.unref(); + } + } +} + +function createShutdownCoordinator(opts) { + return new ShutdownCoordinator(opts); +} + +/** + * Install OS-level signal handlers. Idempotent: second call is a no-op. + * + * @param {ShutdownCoordinator} coordinator + * @param {string[]} signals - signals to handle (default: SIGTERM, SIGINT) + */ +function installSignalHandlers(coordinator, signals) { + if (!coordinator || typeof coordinator.shutdown !== 'function') { + throw new Error('installSignalHandlers: coordinator required'); + } + const sigs = Array.isArray(signals) && signals.length > 0 + ? signals + : ['SIGTERM', 'SIGINT']; + for (const sig of sigs) { + process.on(sig, () => coordinator.shutdown(sig)); + } +} + +module.exports = { + createShutdownCoordinator, + installSignalHandlers, + DEFAULT_DRAIN_TIMEOUT_MS, + ShutdownCoordinator, // exported for tests +};