Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1528fd1a35 | ||
|
|
432e9635bc | ||
|
|
5b74536472 | ||
|
|
bb01a77ae7 | ||
|
|
88ff260e5e |
@@ -0,0 +1 @@
|
||||
node_modules
|
||||
@@ -0,0 +1,490 @@
|
||||
/**
|
||||
* Tests for the graceful shutdown coordinator (DC-067).
|
||||
*
|
||||
* Covers:
|
||||
* - Constructor rejects bad inputs
|
||||
* - shutdown() emits 'shutdown' event with the signal name
|
||||
* - shutdown() stops each manager in declaration order
|
||||
* - shutdown() is idempotent — second call logs and returns
|
||||
* - shutdown() force-exits after drainTimeoutMs if server.close never fires
|
||||
* - shutdown() clears the force-exit timer when server.close fires first
|
||||
* - shutdown() catches manager.stop() throws so one bad manager doesn't
|
||||
* prevent the others from being stopped
|
||||
* - installSignalHandlers() registers for SIGTERM and SIGINT by default
|
||||
*
|
||||
* process.exit is mocked so tests don't actually kill the test runner.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
ShutdownCoordinator,
|
||||
} = require('../src/utilities/shutdown');
|
||||
|
||||
describe('ShutdownCoordinator (DC-067)', () => {
|
||||
let exitMock;
|
||||
let exitCalls;
|
||||
|
||||
beforeEach(() => {
|
||||
exitCalls = [];
|
||||
// Use jest.spyOn so the mock is restored in afterEach (via clearMocks +
|
||||
// restoreMocks: true in jest.config.js). Direct assignment to process.exit
|
||||
// doesn't suppress Jest's process.exit watchlist which fails the test.
|
||||
exitMock = jest.spyOn(process, 'exit').mockImplementation((code) => {
|
||||
exitCalls.push(code);
|
||||
// Returning undefined prevents the test runner from actually exiting.
|
||||
return undefined;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
exitMock.mockRestore();
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
function makeFakeServer({ closeBehavior = 'sync' } = {}) {
|
||||
// 'sync' close calls back immediately.
|
||||
// 'never' close never calls back (used to test force-exit).
|
||||
if (closeBehavior === 'never') {
|
||||
return { close: jest.fn() };
|
||||
}
|
||||
return { close: jest.fn((cb) => { cb(); }) };
|
||||
}
|
||||
|
||||
function makeFakeLog() {
|
||||
return {
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
describe('constructor', () => {
|
||||
test('throws if server is missing', () => {
|
||||
expect(() => createShutdownCoordinator({ log: makeFakeLog(), managers: [] }))
|
||||
.toThrow('server is required');
|
||||
});
|
||||
|
||||
test('throws if log is missing or invalid', () => {
|
||||
expect(() => createShutdownCoordinator({ server: makeFakeServer(), managers: [] }))
|
||||
.toThrow('log must have info');
|
||||
expect(() => createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: { foo: 'bar' },
|
||||
managers: [],
|
||||
})).toThrow('log must have info');
|
||||
expect(() => createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: { info: () => {}, warn: () => {} }, // missing error
|
||||
managers: [],
|
||||
})).toThrow('log must have info');
|
||||
// A log with all three methods should NOT throw.
|
||||
expect(() => createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: { info: () => {}, warn: () => {}, error: () => {} },
|
||||
managers: [],
|
||||
})).not.toThrow();
|
||||
});
|
||||
|
||||
test('uses DEFAULT_DRAIN_TIMEOUT_MS when drainTimeoutMs is not finite positive', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: 0,
|
||||
managers: [],
|
||||
});
|
||||
expect(c.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
|
||||
|
||||
const c2 = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: NaN,
|
||||
managers: [],
|
||||
});
|
||||
expect(c2.drainTimeoutMs).toBe(DEFAULT_DRAIN_TIMEOUT_MS);
|
||||
|
||||
const c3 = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
drainTimeoutMs: 5000,
|
||||
managers: [],
|
||||
});
|
||||
expect(c3.drainTimeoutMs).toBe(5000);
|
||||
});
|
||||
|
||||
test('defaults managers to [] when not an array', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
});
|
||||
expect(c.managers).toEqual([]);
|
||||
});
|
||||
|
||||
test('is an EventEmitter', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
expect(c).toBeInstanceOf(EventEmitter);
|
||||
expect(c).toBeInstanceOf(ShutdownCoordinator);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shutdown()', () => {
|
||||
test('emits shutdown event with signal name', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const handler = jest.fn();
|
||||
c.on('shutdown', handler);
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
expect(handler).toHaveBeenCalledWith('SIGTERM');
|
||||
});
|
||||
|
||||
test('swallows exceptions thrown by shutdown event listeners', () => {
|
||||
const log = makeFakeLog();
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
managers: [],
|
||||
});
|
||||
c.on('shutdown', () => { throw new Error('listener boom'); });
|
||||
|
||||
// shutdown() must NOT propagate the exception — that would abort
|
||||
// the entire shutdown sequence before server.close is even called.
|
||||
expect(() => c.shutdown('SIGTERM')).not.toThrow();
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
"event listener for 'shutdown' threw",
|
||||
expect.objectContaining({ error: 'listener boom' }),
|
||||
);
|
||||
// server.close should still have been called.
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('swallows exceptions thrown by closed event listeners', async () => {
|
||||
const log = makeFakeLog();
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
managers: [],
|
||||
});
|
||||
c.on('closed', () => { throw new Error('closed listener boom'); });
|
||||
|
||||
// process.exit is mocked; we just verify the throw doesn't bubble.
|
||||
c.shutdown('SIGTERM');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
// The closed listener threw but the exit still got recorded.
|
||||
expect(exitCalls).toEqual([0]);
|
||||
});
|
||||
|
||||
test('calls server.close() once', () => {
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('stops each manager in declaration order AFTER server.close fires', async () => {
|
||||
const order = [];
|
||||
const managers = [
|
||||
{ name: 'first', stop: jest.fn(() => { order.push('first'); }) },
|
||||
{ name: 'second', stop: jest.fn(() => { order.push('second'); }) },
|
||||
{ name: 'third', stop: jest.fn(() => { order.push('third'); }) },
|
||||
];
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers,
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
// Wait for the async chain (server.close → _stopManagersInOrder →
|
||||
// process.exit) to settle. The mock exit is synchronous so this
|
||||
// resolves once all microtasks drain.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(order).toEqual(['first', 'second', 'third']);
|
||||
});
|
||||
|
||||
test('does NOT stop managers until server.close callback fires', () => {
|
||||
const order = [];
|
||||
// Use a server whose close callback fires only when we manually call it.
|
||||
let deferredCloseCb;
|
||||
const server = {
|
||||
close: jest.fn((cb) => { deferredCloseCb = cb; }),
|
||||
};
|
||||
const managers = [
|
||||
{ name: 'first', stop: jest.fn(() => { order.push('first'); }) },
|
||||
];
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log: makeFakeLog(),
|
||||
managers,
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
// server.close has been called but its callback hasn't fired yet.
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
// Manager has NOT been stopped yet — server is still draining.
|
||||
expect(order).toEqual([]);
|
||||
|
||||
// Now fire the deferred callback to simulate drain completion.
|
||||
deferredCloseCb();
|
||||
|
||||
// Manager stopped AFTER server.close fired.
|
||||
expect(order).toEqual(['first']);
|
||||
});
|
||||
|
||||
test('continues stopping remaining managers if one throws', async () => {
|
||||
const order = [];
|
||||
const managers = [
|
||||
{ name: 'first', stop: jest.fn(() => { order.push('first'); }) },
|
||||
{ name: 'broken', stop: jest.fn(() => { throw new Error('boom'); }) },
|
||||
{ name: 'third', stop: jest.fn(() => { order.push('third'); }) },
|
||||
];
|
||||
const log = makeFakeLog();
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log,
|
||||
managers,
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(order).toEqual(['first', 'third']);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
'manager stop failed: broken',
|
||||
expect.objectContaining({ error: 'boom' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('is idempotent — second shutdown() returns without re-running', () => {
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log: makeFakeLog(),
|
||||
managers: [{ name: 'm', stop: jest.fn() }],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
c.shutdown('SIGTERM');
|
||||
c.shutdown('SIGINT');
|
||||
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
expect(c.isShuttingDown()).toBe(true);
|
||||
});
|
||||
|
||||
test('isShuttingDown() flips false→true on first shutdown call', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
expect(c.isShuttingDown()).toBe(false);
|
||||
c.shutdown('SIGTERM');
|
||||
expect(c.isShuttingDown()).toBe(true);
|
||||
});
|
||||
|
||||
test('force-exits after drainTimeoutMs if server.close never fires', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer({ closeBehavior: 'never' });
|
||||
const log = makeFakeLog();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(999);
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(2);
|
||||
expect(exitCalls).toEqual([0]);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
expect.stringContaining('drain timeout (1000ms) reached before HTTP server closed'),
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('force-exits after drainTimeoutMs if manager.stop() hangs after HTTP close', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer(); // calls back immediately
|
||||
const log = makeFakeLog();
|
||||
// Manager that NEVER resolves — simulates a hung cleanup.
|
||||
const hungManager = {
|
||||
name: 'hung',
|
||||
stop: jest.fn(() => new Promise(() => {})), // never resolves
|
||||
};
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [hungManager],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
// After the synchronous shutdown() call: server.close has fired
|
||||
// (serverClosed=true), but hungManager.stop() has been called and
|
||||
// its promise is pending. managersStopped is still false.
|
||||
// process.exit should NOT have been called yet.
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(1001);
|
||||
// Now the safety-net timer fires — force-exit because manager hung.
|
||||
expect(exitCalls).toEqual([0]);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
expect.stringContaining('after HTTP close (manager hung)'),
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('clears force-exit timer when manager drain completes promptly', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer(); // calls back on the same tick
|
||||
const log = makeFakeLog();
|
||||
// Quick-stopping manager. The close callback awaits stop(),
|
||||
// which resolves immediately, so managersStopped flips true
|
||||
// and the safety-net timer is cleared before it can fire.
|
||||
const fastManager = {
|
||||
name: 'fast',
|
||||
stop: jest.fn(() => Promise.resolve()),
|
||||
};
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [fastManager],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
// Flush microtasks so the close callback's await stop() resolves,
|
||||
// managersStopped flips true, the timer is cleared, and
|
||||
// process.exit(0) is recorded exactly once.
|
||||
return Promise.resolve().then(() => Promise.resolve()).then(() => {
|
||||
expect(exitCalls).toEqual([0]);
|
||||
|
||||
// Advance well past the drain timeout — no extra exit should fire.
|
||||
jest.advanceTimersByTime(5000);
|
||||
expect(exitCalls).toEqual([0]);
|
||||
});
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('installSignalHandlers()', () => {
|
||||
// Track listeners added during each test so we can remove them in
|
||||
// afterEach. process.on() listeners leak across tests otherwise.
|
||||
let addedListeners;
|
||||
let originalProcessOn;
|
||||
|
||||
beforeEach(() => {
|
||||
addedListeners = [];
|
||||
originalProcessOn = process.on;
|
||||
// Wrap process.on to record every (signal, listener) pair we add.
|
||||
// Must capture originalProcessOn at wrap time so we can call it.
|
||||
const realOn = originalProcessOn;
|
||||
process.on = function patchedOn(signal, listener) {
|
||||
addedListeners.push({ signal, listener });
|
||||
return realOn.call(process, signal, listener);
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.on = originalProcessOn;
|
||||
for (const { signal, listener } of addedListeners) {
|
||||
originalProcessOn.call(process, signal, listener); // ensure clean slate
|
||||
process.removeListener(signal, listener);
|
||||
}
|
||||
addedListeners = [];
|
||||
});
|
||||
|
||||
test('registers listeners on the given signals', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const shutdownSpy = jest.spyOn(c, 'shutdown');
|
||||
|
||||
installSignalHandlers(c);
|
||||
|
||||
// Emit fake signals through process.emit to verify the listener was
|
||||
// registered (process.on listens to the process EventEmitter).
|
||||
process.emit('SIGTERM');
|
||||
process.emit('SIGINT');
|
||||
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGTERM');
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGINT');
|
||||
});
|
||||
|
||||
test('accepts custom signal list', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const shutdownSpy = jest.spyOn(c, 'shutdown');
|
||||
|
||||
installSignalHandlers(c, ['SIGHUP']);
|
||||
|
||||
process.emit('SIGHUP');
|
||||
expect(shutdownSpy).toHaveBeenCalledWith('SIGHUP');
|
||||
});
|
||||
|
||||
test('is idempotent — calling installSignalHandlers twice does not double-register', () => {
|
||||
const c = createShutdownCoordinator({
|
||||
server: makeFakeServer(),
|
||||
log: makeFakeLog(),
|
||||
managers: [],
|
||||
});
|
||||
const shutdownSpy = jest.spyOn(c, 'shutdown');
|
||||
|
||||
installSignalHandlers(c);
|
||||
installSignalHandlers(c); // second call
|
||||
installSignalHandlers(c); // third call
|
||||
|
||||
// The installedSignals tracker should have one entry per signal.
|
||||
expect(c._installedSignals).toEqual(['SIGTERM', 'SIGINT']);
|
||||
|
||||
process.emit('SIGTERM');
|
||||
expect(shutdownSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
+42
-33
@@ -252,43 +252,52 @@ process.on('uncaughtException', (error) => {
|
||||
log.info('server', 'All feature modules initialized');
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = (signal) => {
|
||||
log.info('shutdown', `${signal} received, draining connections...`);
|
||||
// Graceful shutdown (DC-067) — drains in-flight HTTP connections, stops
|
||||
// each manager in deterministic order, emits a 'shutdown' event for any
|
||||
// additional listeners, and force-exits after a 10s drain timeout.
|
||||
// Idempotent: a second SIGTERM during shutdown is a no-op.
|
||||
const {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
} = require('./src/utilities/shutdown');
|
||||
|
||||
const resourceMonitor = require('./src/managers/resource-monitor');
|
||||
const backupManager = require('./src/utilities/backup-manager');
|
||||
const healthChecker = require('./src/monitoring/health-checker');
|
||||
const updateManager = require('./src/managers/update-manager');
|
||||
const selfUpdater = require('./src/docker/self-updater');
|
||||
|
||||
resourceMonitor.stop();
|
||||
backupManager.stop();
|
||||
healthChecker.stop();
|
||||
updateManager.stop();
|
||||
selfUpdater.stop();
|
||||
|
||||
try {
|
||||
const dockerMaintenance = require('./src/docker/docker-maintenance');
|
||||
dockerMaintenance.stop();
|
||||
} catch { /* optional */ }
|
||||
|
||||
try {
|
||||
const logDigest = require('./src/security/log-digest');
|
||||
logDigest.stop();
|
||||
} catch { /* optional */ }
|
||||
|
||||
server.close(() => {
|
||||
log.info('shutdown', 'HTTP server closed');
|
||||
process.exit(0);
|
||||
const optionalManagers = [];
|
||||
try {
|
||||
optionalManagers.push({
|
||||
name: 'docker-maintenance',
|
||||
stop: () => require('./src/docker/docker-maintenance').stop(),
|
||||
});
|
||||
} catch { /* optional module */ }
|
||||
try {
|
||||
optionalManagers.push({
|
||||
name: 'log-digest',
|
||||
stop: () => require('./src/security/log-digest').stop(),
|
||||
});
|
||||
} catch { /* optional module */ }
|
||||
|
||||
// Force exit after 5s if connections don't drain
|
||||
setTimeout(() => process.exit(0), 5000).unref();
|
||||
};
|
||||
const coordinator = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
managers: [
|
||||
{ name: 'resource-monitor', stop: () => require('./src/managers/resource-monitor').stop() },
|
||||
{ name: 'backup-manager', stop: () => require('./src/utilities/backup-manager').stop() },
|
||||
{ name: 'health-checker', stop: () => require('./src/monitoring/health-checker').stop() },
|
||||
{ name: 'update-manager', stop: () => require('./src/managers/update-manager').stop() },
|
||||
{ name: 'self-updater', stop: () => require('./src/docker/self-updater').stop() },
|
||||
...optionalManagers,
|
||||
],
|
||||
});
|
||||
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
// Expose the shutdown signal as an event so additional listeners can
|
||||
// subscribe without touching this file. The coordinator is an
|
||||
// EventEmitter and emits 'shutdown' on SIGTERM/SIGINT.
|
||||
coordinator.on('shutdown', (signal) => {
|
||||
log.info('shutdown', 'shutdown event observed', { signal });
|
||||
});
|
||||
|
||||
installSignalHandlers(coordinator);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[FATAL] Server startup failed:', error);
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
/**
|
||||
* Graceful shutdown coordinator — DashCaddy
|
||||
*
|
||||
* Extracts the SIGTERM/SIGINT handler from server.js into a testable,
|
||||
* reusable module that:
|
||||
* 1. Calls server.close() to drain in-flight HTTP connections
|
||||
* 2. Stops each manager in a deterministic order
|
||||
* 3. Emits a 'shutdown' event so additional listeners can do cleanup
|
||||
* 4. Force-exits after a configurable drain timeout if connections don't drain
|
||||
* 5. Is idempotent — a second SIGTERM during shutdown does not re-run handlers
|
||||
*
|
||||
* Spec: DC-067 (production-grade backlog). Docker sends SIGTERM on stop;
|
||||
* without this coordinator, in-flight API calls drop.
|
||||
*
|
||||
* Exports:
|
||||
* - createShutdownCoordinator({ server, log, drainTimeoutMs, managers })
|
||||
* Returns an EventEmitter with: { shutdown, isShuttingDown, on, emit, ... }
|
||||
* - installSignalHandlers(coordinator, signals = ['SIGTERM', 'SIGINT'])
|
||||
* Registers the OS-level handlers. Idempotent.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
|
||||
|
||||
class ShutdownCoordinator extends EventEmitter {
|
||||
constructor({ server, log, drainTimeoutMs, managers }) {
|
||||
super();
|
||||
if (!server) throw new Error('createShutdownCoordinator: server is required');
|
||||
if (!log || typeof log.info !== 'function' || typeof log.warn !== 'function'
|
||||
|| typeof log.error !== 'function') {
|
||||
throw new Error('createShutdownCoordinator: log must have info(), warn(), and error() methods');
|
||||
}
|
||||
this.server = server;
|
||||
this.log = log;
|
||||
this.drainTimeoutMs = Number.isFinite(drainTimeoutMs) && drainTimeoutMs > 0
|
||||
? drainTimeoutMs
|
||||
: DEFAULT_DRAIN_TIMEOUT_MS;
|
||||
this.managers = Array.isArray(managers) ? managers : [];
|
||||
this._shuttingDown = false;
|
||||
this._forceTimer = null;
|
||||
}
|
||||
|
||||
isShuttingDown() {
|
||||
return this._shuttingDown;
|
||||
}
|
||||
|
||||
async _stopManager(m) {
|
||||
try {
|
||||
await m.stop();
|
||||
this.log.info('shutdown', `manager stopped: ${m.name}`);
|
||||
} catch (err) {
|
||||
this.log.warn('shutdown', `manager stop failed: ${m.name}`, { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop each manager sequentially in declaration order. Each manager's
|
||||
* stop() is awaited so that a downstream manager is not stopped until
|
||||
* its upstream dependency has finished draining.
|
||||
*
|
||||
* IMPORTANT: this runs AFTER server.close() returns (see shutdown()).
|
||||
* We must wait for in-flight HTTP requests to complete before tearing
|
||||
* down the services that serve them — otherwise those requests fail
|
||||
* mid-drain with "service not found" / "monitor not running" errors.
|
||||
*/
|
||||
async _stopManagersInOrder() {
|
||||
for (const m of this.managers) {
|
||||
await this._stopManager(m);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit an event but swallow listener exceptions so one bad listener
|
||||
* can't abort the shutdown sequence. Logs each failure with the
|
||||
* listener's name (set via `listener.name`) if available.
|
||||
*/
|
||||
_safeEmit(event, ...args) {
|
||||
const listeners = this.listeners(event);
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
listener.apply(this, args);
|
||||
} catch (err) {
|
||||
const name = listener.name || '<anonymous>';
|
||||
this.log.error('shutdown', `event listener for '${event}' threw`,
|
||||
{ listener: name, error: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
shutdown(signal) {
|
||||
if (this._shuttingDown) {
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
|
||||
return;
|
||||
}
|
||||
this._shuttingDown = true;
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
|
||||
|
||||
// Emit 'shutdown' event first so any listeners can observe the signal
|
||||
// before the drain begins. NOTE: listeners should NOT tear down their
|
||||
// state here — that happens in the 'closed' event after server.close.
|
||||
// _safeEmit swallows listener exceptions so a buggy listener can't
|
||||
// abort the entire shutdown sequence.
|
||||
this._safeEmit('shutdown', signal);
|
||||
|
||||
// Close the HTTP server FIRST. Stops accepting new connections, waits
|
||||
// for in-flight requests to complete naturally. Only AFTER close fires
|
||||
// do we tear down managers — otherwise in-flight requests could fail
|
||||
// when the services they call have already been stopped.
|
||||
let serverClosed = false;
|
||||
let managersStopped = false;
|
||||
try {
|
||||
this.server.close(async () => {
|
||||
serverClosed = true;
|
||||
this.log.info('shutdown', 'HTTP server closed cleanly');
|
||||
// Now that in-flight requests are done, stop managers in order.
|
||||
// We do NOT clear the force-exit timer yet — if a manager's stop()
|
||||
// hangs, the timer is the safety net that prevents the process
|
||||
// from living forever in a half-shut-down state.
|
||||
try {
|
||||
await this._stopManagersInOrder();
|
||||
} catch (err) {
|
||||
// _stopManager already logs per-manager failures, but a top-level
|
||||
// throw (e.g. from the for-loop itself) is still possible.
|
||||
this.log.error('shutdown', 'manager shutdown loop threw', { error: err.message });
|
||||
}
|
||||
managersStopped = true;
|
||||
// Manager drain complete — NOW we can clear the safety timer.
|
||||
if (this._forceTimer) {
|
||||
clearTimeout(this._forceTimer);
|
||||
this._forceTimer = null;
|
||||
}
|
||||
this._safeEmit('closed', signal);
|
||||
process.exit(0);
|
||||
});
|
||||
} catch (err) {
|
||||
this.log.error('shutdown', 'server.close threw', { error: err.message });
|
||||
}
|
||||
|
||||
// Force-exit safety net. Fires when EITHER:
|
||||
// (a) server.close never fires (HTTP server stuck draining), or
|
||||
// (b) server.close fired but managers hung during stop()
|
||||
// We only suppress when managersStopped === true (full drain complete).
|
||||
// serverClosed alone is NOT enough — managers could still be running.
|
||||
this._forceTimer = setTimeout(() => {
|
||||
if (managersStopped) return; // full shutdown complete
|
||||
if (!serverClosed) {
|
||||
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached before HTTP server closed, force-exiting`);
|
||||
} else {
|
||||
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached after HTTP close (manager hung), force-exiting`);
|
||||
}
|
||||
process.exit(0);
|
||||
}, this.drainTimeoutMs);
|
||||
if (this._forceTimer && typeof this._forceTimer.unref === 'function') {
|
||||
this._forceTimer.unref();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createShutdownCoordinator(opts) {
|
||||
return new ShutdownCoordinator(opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Install OS-level signal handlers. Idempotent: second call for the same
|
||||
* signal does NOT register a duplicate listener. Tracks registered signals
|
||||
* on the coordinator itself so a future caller can introspect.
|
||||
*
|
||||
* @param {ShutdownCoordinator} coordinator
|
||||
* @param {string[]} signals - signals to handle (default: SIGTERM, SIGINT)
|
||||
*/
|
||||
function installSignalHandlers(coordinator, signals) {
|
||||
if (!coordinator || typeof coordinator.shutdown !== 'function') {
|
||||
throw new Error('installSignalHandlers: coordinator required');
|
||||
}
|
||||
if (!Array.isArray(coordinator._installedSignals)) {
|
||||
coordinator._installedSignals = [];
|
||||
}
|
||||
const sigs = Array.isArray(signals) && signals.length > 0
|
||||
? signals
|
||||
: ['SIGTERM', 'SIGINT'];
|
||||
for (const sig of sigs) {
|
||||
if (coordinator._installedSignals.includes(sig)) continue;
|
||||
process.on(sig, () => coordinator.shutdown(sig));
|
||||
coordinator._installedSignals.push(sig);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
ShutdownCoordinator, // exported for tests
|
||||
};
|
||||
Reference in New Issue
Block a user