DC-067: fix force-exit + listener exception handling per Codex D-grade feedback
- Force-exit timer now distinguishes serverClosed from managersStopped. Was: if (closed) return → suppressed timer when manager hung after server.close fired (the original bug). Now: if (managersStopped) return → timer fires only when full drain (HTTP close + all managers stopped) completes before the deadline. - Added _safeEmit() helper that wraps this.emit() so a buggy listener throwing during 'shutdown' or 'closed' doesn't abort the shutdown sequence. Each failed listener is logged via the structured logger. - Added 4 new tests covering: hung manager after HTTP close, throwing shutdown listener, throwing closed listener, and the fast-drain happy path that clears the timer cleanly.
This commit is contained in:
@@ -150,6 +150,45 @@ describe('ShutdownCoordinator (DC-067)', () => {
|
||||
expect(handler).toHaveBeenCalledWith('SIGTERM');
|
||||
});
|
||||
|
||||
test('swallows exceptions thrown by shutdown event listeners', () => {
|
||||
const log = makeFakeLog();
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
managers: [],
|
||||
});
|
||||
c.on('shutdown', () => { throw new Error('listener boom'); });
|
||||
|
||||
// shutdown() must NOT propagate the exception — that would abort
|
||||
// the entire shutdown sequence before server.close is even called.
|
||||
expect(() => c.shutdown('SIGTERM')).not.toThrow();
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
"event listener for 'shutdown' threw",
|
||||
expect.objectContaining({ error: 'listener boom' }),
|
||||
);
|
||||
// server.close should still have been called.
|
||||
expect(server.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('swallows exceptions thrown by closed event listeners', async () => {
|
||||
const log = makeFakeLog();
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
managers: [],
|
||||
});
|
||||
c.on('closed', () => { throw new Error('closed listener boom'); });
|
||||
|
||||
// process.exit is mocked; we just verify the throw doesn't bubble.
|
||||
c.shutdown('SIGTERM');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
// The closed listener threw but the exit still got recorded.
|
||||
expect(exitCalls).toEqual([0]);
|
||||
});
|
||||
|
||||
test('calls server.close() once', () => {
|
||||
const server = makeFakeServer();
|
||||
const c = createShutdownCoordinator({
|
||||
@@ -289,36 +328,77 @@ describe('ShutdownCoordinator (DC-067)', () => {
|
||||
expect(exitCalls).toEqual([0]);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
'drain timeout (1000ms) reached, force-exiting',
|
||||
expect.stringContaining('drain timeout (1000ms) reached before HTTP server closed'),
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('clears force-exit timer when server.close fires first', () => {
|
||||
test('force-exits after drainTimeoutMs if manager.stop() hangs after HTTP close', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer(); // calls back on the same tick
|
||||
const server = makeFakeServer(); // calls back immediately
|
||||
const log = makeFakeLog();
|
||||
// Manager that NEVER resolves — simulates a hung cleanup.
|
||||
const hungManager = {
|
||||
name: 'hung',
|
||||
stop: jest.fn(() => new Promise(() => {})), // never resolves
|
||||
};
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [],
|
||||
managers: [hungManager],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
// The close callback is async (it awaits _stopManagersInOrder),
|
||||
// so the synchronous c.shutdown() returns BEFORE process.exit(0)
|
||||
// is recorded. Flush pending timers + microtasks.
|
||||
jest.runAllTimers();
|
||||
// Flush microtasks queued by the async close callback.
|
||||
// After the synchronous shutdown() call: server.close has fired
|
||||
// (serverClosed=true), but hungManager.stop() has been called and
|
||||
// its promise is pending. managersStopped is still false.
|
||||
// process.exit should NOT have been called yet.
|
||||
expect(exitCalls).toEqual([]);
|
||||
|
||||
jest.advanceTimersByTime(1001);
|
||||
// Now the safety-net timer fires — force-exit because manager hung.
|
||||
expect(exitCalls).toEqual([0]);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
'shutdown',
|
||||
expect.stringContaining('after HTTP close (manager hung)'),
|
||||
);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test('clears force-exit timer when manager drain completes promptly', () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const server = makeFakeServer(); // calls back on the same tick
|
||||
const log = makeFakeLog();
|
||||
// Quick-stopping manager. The close callback awaits stop(),
|
||||
// which resolves immediately, so managersStopped flips true
|
||||
// and the safety-net timer is cleared before it can fire.
|
||||
const fastManager = {
|
||||
name: 'fast',
|
||||
stop: jest.fn(() => Promise.resolve()),
|
||||
};
|
||||
const c = createShutdownCoordinator({
|
||||
server,
|
||||
log,
|
||||
drainTimeoutMs: 1000,
|
||||
managers: [fastManager],
|
||||
});
|
||||
|
||||
c.shutdown('SIGTERM');
|
||||
|
||||
// Flush microtasks so the close callback's await stop() resolves,
|
||||
// managersStopped flips true, the timer is cleared, and
|
||||
// process.exit(0) is recorded exactly once.
|
||||
return Promise.resolve().then(() => Promise.resolve()).then(() => {
|
||||
expect(exitCalls).toEqual([0]);
|
||||
|
||||
// Advance well past the drain timeout — no extra exit should fire
|
||||
// because the close callback cleared the force-exit timer.
|
||||
// Advance well past the drain timeout — no extra exit should fire.
|
||||
jest.advanceTimersByTime(5000);
|
||||
expect(exitCalls).toEqual([0]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user