DC-067: graceful shutdown coordinator (EventEmitter, 10s drain, idempotent)
This commit is contained in:
hermes
@@ -0,0 +1,318 @@
|
||||
/**
|
||||
* Tests for the graceful shutdown coordinator (DC-067).
|
||||
*
|
||||
* Covers:
|
||||
* - Constructor rejects bad inputs
|
||||
* - shutdown() emits 'shutdown' event with the signal name
|
||||
* - shutdown() stops each manager in declaration order
|
||||
* - shutdown() is idempotent — second call logs and returns
|
||||
* - shutdown() force-exits after drainTimeoutMs if server.close never fires
|
||||
* - shutdown() clears the force-exit timer when server.close fires first
|
||||
* - shutdown() catches manager.stop() throws so one bad manager doesn't
|
||||
* prevent the others from being stopped
|
||||
* - installSignalHandlers() registers for SIGTERM and SIGINT by default
|
||||
*
|
||||
* process.exit is mocked so tests don't actually kill the test runner.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
ShutdownCoordinator,
|
||||
} = require('../src/utilities/shutdown');
|
||||
|
||||
describe('ShutdownCoordinator (DC-067)', () => {
|
||||
let exitMock;
|
||||
let exitCalls;
|
||||
|
||||
beforeEach(() => {
|
||||
exitCalls = [];
|
||||
// Use jest.spyOn so the mock is restored in afterEach (via clearMocks +
|
||||
// restoreMocks: true in jest.config.js). Direct assignment to process.exit
|
||||
// doesn't suppress Jest's process.exit watchlist which fails the test.
|
||||
exitMock = jest.spyOn(process, 'exit').mockImplementation((code) => {
|
||||
exitCalls.push(code);
|
||||
// Returning undefined prevents the test runner from actually exiting.
|
||||
return undefined;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
exitMock.mockRestore();
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
function makeFakeServer() {
|
||||
// Synchronous close callback. async (setImmediate) callbacks would fire
|
||||
// AFTER the test that triggered shutdown has ended, hitting the real
|
||||
// process.exit in the restored mock window.
|
||||
return { close: jest.fn((cb) => { cb(); }) };
|
||||
}
|
||||
|
||||
function makeFakeLog() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
describe('constructor', () => {
|
||||
test('throws if server is missing', () => {
|
||||
expect(() => createShutdownCoordinator({ log: makeFakeLog(), managers: [] }))
|
||||
.toThrow('server is required');
|
||||
});
|
||||
|
||||
test('throws if log is missing or invalid', () => {
|
||||
expect(() => createShutdownCoordinator({ server: makeFakeServer(), managers: [] }))
|
||||
.toThrow('log with info/warn is required');
|
||||
expect(() => createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: { foo: 'bar' },
|
||||
managers: [],
|
||||
})).toThrow('log with info/warn is required');
|
||||
});
|
||||
|
||||
test('uses DEFAULT_DRAIN_TIMEOUT_MS when drainTimeoutMs is not finite positive', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: 0,
|
||||
managers: [],
|
||||
});
|
||||
expect(c.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
|
||||
|
||||
const c2 = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: NaN,
|
||||
managers: [],
|
||||
});
|
||||
expect(c2.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
|
||||
|
||||
const c3 = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: 5000,
|
||||
managers: [],
|
||||
});
|
||||
expect(c3.drainTimeoutMs).toBe(5000);
|
||||
});
|
||||
|
||||
test('defaults managers to [] when not an array', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
});
|
||||
expect(c.managers).toEqual([]);
|
||||
});
|
||||
|
||||
test('is an EventEmitter', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
expect(c).toBeInstanceOf(EventEmitter);
|
||||
expect(c).toBeInstanceOf(ShutdownCoordinator);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shutdown()', () => {
|
||||
test('emits shutdown event with signal name', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const handler = jest.fn();
|
||||
c.on('shutdown', handler);
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
expect(handler).toHaveBeenCalledWith('SIGTERM');
|
||||
});
|
||||
|
||||
test('calls server.close() once', () => {
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('stops each manager in declaration order', async () => {
|
||||
const order = [];
|
||||
const managers = [
|
||||
{ name: 'first', stop: jest.fn(() => { order.push('first'); }) },
|
||||
{ name: 'second', stop: jest.fn(() => { order.push('second'); }) },
|
||||
{ name: 'third', stop: jest.fn(() => { order.push('third'); }) },
|
||||
];
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers,
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
// Wait one microtask + macrotask for the async chain to settle.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(order).toEqual(['first', 'second', 'third']);
|
||||
});
|
||||
|
||||
test('continues stopping remaining managers if one throws', async () => {
|
||||
const order = [];
|
||||
const managers = [
|
||||
{ name: 'first', stop: jest.fn(() => { order.push('first'); }) },
|
||||
{ name: 'broken', stop: jest.fn(() => { throw new Error('boom'); }) },
|
||||
{ name: 'third', stop: jest.fn(() => { order.push('third'); }) },
|
||||
];
|
||||
const log = makeFakeLog();
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log,
|
||||
managers,
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(order).toEqual(['first', 'third']);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
'manager stop failed: broken',
|
||||
expect.objectContaining({ error: 'boom' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('is idempotent — second shutdown() returns without re-running', () => {
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log: makeFakeLog(),
|
||||
managers: [{ name: 'm', stop: jest.fn() }],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
c.shutdown('SIGTERM');
|
||||
c.shutdown('SIGINT');
|
||||
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
expect(c.isShuttingDown()).toBe(true);
|
||||
});
|
||||
|
||||
test('isShuttingDown() flips false→true on first shutdown call', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
expect(c.isShuttingDown()).toBe(false);
|
||||
c.shutdown('SIGTERM');
|
||||
expect(c.isShuttingDown()).toBe(true);
|
||||
});
|
||||
|
||||
test('force-exits after drainTimeoutMs if server.close never fires', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = { close: jest.fn(/* never calls back */) };
|
||||
const log = makeFakeLog();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(999);
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(2);
|
||||
expect(exitCalls).toEqual([0]);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
'drain timeout (1000ms) reached, force-exiting',
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('clears force-exit timer when server.close fires first', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer(); // calls back on setImmediate
|
||||
const log = makeFakeLog();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
// Flush the setImmediate so server.close callback fires.
|
||||
jest.runOnlyPendingTimers();
|
||||
jest.advanceTimersByTime(0);
|
||||
|
||||
expect(exitCalls).toEqual([0]);
|
||||
|
||||
// Even after the full drain timeout, no extra exit should fire
|
||||
// (because we cleared the timer in the close callback).
|
||||
jest.advanceTimersByTime(5000);
|
||||
expect(exitCalls).toEqual([0]);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('installSignalHandlers()', () => {
|
||||
test('registers listeners on the given signals', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const shutdownSpy = jest.spyOn(c, 'shutdown');
|
||||
|
||||
installSignalHandlers(c);
|
||||
|
||||
// Emit fake signals through process.emit to verify the listener was
|
||||
// registered (process.on listens to the process EventEmitter).
|
||||
process.emit('SIGTERM');
|
||||
process.emit('SIGINT');
|
||||
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGTERM');
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGINT');
|
||||
});
|
||||
|
||||
test('accepts custom signal list', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const shutdownSpy = jest.spyOn(c, 'shutdown');
|
||||
|
||||
installSignalHandlers(c, ['SIGHUP']);
|
||||
|
||||
process.emit('SIGHUP');
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGHUP');
|
||||
});
|
||||
});
|
||||
});
|
||||
+43
-34
@@ -252,43 +252,52 @@ process.on('uncaughtException', (error) => {
|
||||
log.info('server', 'All feature modules initialized');
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = (signal) => {
|
||||
log.info('shutdown', `${signal} received, draining connections...`);
|
||||
|
||||
const resourceMonitor = require('./src/managers/resource-monitor');
|
||||
const backupManager = require('./src/utilities/backup-manager');
|
||||
const healthChecker = require('./src/monitoring/health-checker');
|
||||
const updateManager = require('./src/managers/update-manager');
|
||||
const selfUpdater = require('./src/docker/self-updater');
|
||||
|
||||
resourceMonitor.stop();
|
||||
backupManager.stop();
|
||||
healthChecker.stop();
|
||||
updateManager.stop();
|
||||
selfUpdater.stop();
|
||||
|
||||
try {
|
||||
const dockerMaintenance = require('./src/docker/docker-maintenance');
|
||||
dockerMaintenance.stop();
|
||||
} catch { /* optional */ }
|
||||
|
||||
try {
|
||||
const logDigest = require('./src/security/log-digest');
|
||||
logDigest.stop();
|
||||
} catch { /* optional */ }
|
||||
// Graceful shutdown (DC-067) — drains in-flight HTTP connections, stops
|
||||
// each manager in deterministic order, emits a 'shutdown' event for any
|
||||
// additional listeners, and force-exits after a 10s drain timeout.
|
||||
// Idempotent: a second SIGTERM during shutdown is a no-op.
|
||||
const {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
} = require('./src/utilities/shutdown');
|
||||
|
||||
server.close(() => {
|
||||
log.info('shutdown', 'HTTP server closed');
|
||||
process.exit(0);
|
||||
const optionalManagers = [];
|
||||
try {
|
||||
optionalManagers.push({
|
||||
name: 'docker-maintenance',
|
||||
stop: () => require('./src/docker/docker-maintenance').stop(),
|
||||
});
|
||||
|
||||
// Force exit after 5s if connections don't drain
|
||||
setTimeout(() => process.exit(0), 5000).unref();
|
||||
};
|
||||
} catch { /* optional module */ }
|
||||
try {
|
||||
optionalManagers.push({
|
||||
name: 'log-digest',
|
||||
stop: () => require('./src/security/log-digest').stop(),
|
||||
});
|
||||
} catch { /* optional module */ }
|
||||
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
const coordinator = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
managers: [
|
||||
{ name: 'resource-monitor', stop: () => require('./src/managers/resource-monitor').stop() },
|
||||
{ name: 'backup-manager', stop: () => require('./src/utilities/backup-manager').stop() },
|
||||
{ name: 'health-checker', stop: () => require('./src/monitoring/health-checker').stop() },
|
||||
{ name: 'update-manager', stop: () => require('./src/managers/update-manager').stop() },
|
||||
{ name: 'self-updater', stop: () => require('./src/docker/self-updater').stop() },
|
||||
...optionalManagers,
|
||||
],
|
||||
});
|
||||
|
||||
// Expose the shutdown signal as an event so additional listeners can
|
||||
// subscribe without touching this file. The coordinator is an
|
||||
// EventEmitter and emits 'shutdown' on SIGTERM/SIGINT.
|
||||
coordinator.on('shutdown', (signal) => {
|
||||
log.info('shutdown', 'shutdown event observed', { signal });
|
||||
});
|
||||
|
||||
installSignalHandlers(coordinator);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[FATAL] Server startup failed:', error);
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Graceful shutdown coordinator — DashCaddy
|
||||
*
|
||||
* Extracts the SIGTERM/SIGINT handler from server.js into a testable,
|
||||
* reusable module that:
|
||||
* 1. Calls server.close() to drain in-flight HTTP connections
|
||||
* 2. Stops each manager in a deterministic order
|
||||
* 3. Emits a 'shutdown' event so additional listeners can do cleanup
|
||||
* 4. Force-exits after a configurable drain timeout if connections don't drain
|
||||
* 5. Is idempotent — a second SIGTERM during shutdown does not re-run handlers
|
||||
*
|
||||
* Spec: DC-067 (production-grade backlog). Docker sends SIGTERM on stop;
|
||||
* without this coordinator, in-flight API calls drop.
|
||||
*
|
||||
* Exports:
|
||||
* - createShutdownCoordinator({ server, log, drainTimeoutMs, managers })
|
||||
* Returns an EventEmitter with: { shutdown, isShuttingDown, on, emit, ... }
|
||||
* - installSignalHandlers(coordinator, signals = ['SIGTERM', 'SIGINT'])
|
||||
* Registers the OS-level handlers. Idempotent.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
|
||||
|
||||
class ShutdownCoordinator extends EventEmitter {
|
||||
constructor({ server, log, drainTimeoutMs, managers }) {
|
||||
super();
|
||||
if (!server) throw new Error('createShutdownCoordinator: server is required');
|
||||
if (!log || typeof log.info !== 'function') {
|
||||
throw new Error('createShutdownCoordinator: log with info/warn is required');
|
||||
}
|
||||
this.server = server;
|
||||
this.log = log;
|
||||
this.drainTimeoutMs = Number.isFinite(drainTimeoutMs) && drainTimeoutMs > 0
|
||||
? drainTimeoutMs
|
||||
: DEFAULT_DRAIN_TIMEOUT_MS;
|
||||
this.managers = Array.isArray(managers) ? managers : [];
|
||||
this._shuttingDown = false;
|
||||
this._forceTimer = null;
|
||||
}
|
||||
|
||||
isShuttingDown() {
|
||||
return this._shuttingDown;
|
||||
}
|
||||
|
||||
async _stopManager(m) {
|
||||
try {
|
||||
await m.stop();
|
||||
this.log.info('shutdown', `manager stopped: ${m.name}`);
|
||||
} catch (err) {
|
||||
this.log.warn('shutdown', `manager stop failed: ${m.name}`, { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
shutdown(signal) {
|
||||
if (this._shuttingDown) {
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
|
||||
return;
|
||||
}
|
||||
this._shuttingDown = true;
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
|
||||
|
||||
// Emit 'shutdown' event first so any listeners can flush their state
|
||||
// before the managers start stopping.
|
||||
this.emit('shutdown', signal);
|
||||
|
||||
// Stop each manager in order. Fire-and-forget — each manager's stop()
|
||||
// is expected to be fast (cancel timers, flush buffers). If a manager
|
||||
// has long async work, it should expose its own drain mechanism.
|
||||
for (const m of this.managers) {
|
||||
this._stopManager(m);
|
||||
}
|
||||
|
||||
// Close the HTTP server. Stops accepting new connections, waits for
|
||||
// in-flight requests to complete naturally.
|
||||
let closed = false;
|
||||
try {
|
||||
this.server.close(() => {
|
||||
closed = true;
|
||||
this.log.info('shutdown', 'HTTP server closed cleanly');
|
||||
if (this._forceTimer) {
|
||||
clearTimeout(this._forceTimer);
|
||||
this._forceTimer = null;
|
||||
}
|
||||
this.emit('closed', signal);
|
||||
process.exit(0);
|
||||
});
|
||||
} catch (err) {
|
||||
this.log.error('shutdown', 'server.close threw', { error: err.message });
|
||||
}
|
||||
|
||||
// Force-exit if drain doesn't complete in time.
|
||||
this._forceTimer = setTimeout(() => {
|
||||
if (closed) return; // race: server.close fired AND timer fired
|
||||
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`);
|
||||
process.exit(0);
|
||||
}, this.drainTimeoutMs);
|
||||
if (this._forceTimer && typeof this._forceTimer.unref === 'function') {
|
||||
this._forceTimer.unref();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createShutdownCoordinator(opts) {
|
||||
return new ShutdownCoordinator(opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Install OS-level signal handlers. Idempotent: second call is a no-op.
|
||||
*
|
||||
* @param {ShutdownCoordinator} coordinator
|
||||
* @param {string[]} signals - signals to handle (default: SIGTERM, SIGINT)
|
||||
*/
|
||||
function installSignalHandlers(coordinator, signals) {
|
||||
if (!coordinator || typeof coordinator.shutdown !== 'function') {
|
||||
throw new Error('installSignalHandlers: coordinator required');
|
||||
}
|
||||
const sigs = Array.isArray(signals) && signals.length > 0
|
||||
? signals
|
||||
: ['SIGTERM', 'SIGINT'];
|
||||
for (const sig of sigs) {
|
||||
process.on(sig, () => coordinator.shutdown(sig));
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
ShutdownCoordinator, // exported for tests
|
||||
};
|
||||
Reference in New Issue
Block a user