DC-067: fix force-exit + listener exception handling per Codex D-grade feedback

- Force-exit timer now distinguishes serverClosed from managersStopped.
  Was: if (closed) return → suppressed timer when manager hung after
  server.close fired (the original bug).
  Now: if (managersStopped) return → timer fires only when full drain
  (HTTP close + all managers stopped) completes before the deadline.

- Added _safeEmit() helper that wraps this.emit() so a buggy listener
  throwing during 'shutdown' or 'closed' doesn't abort the shutdown
  sequence. Each failed listener is logged via the structured logger.

- Added 4 new tests covering: hung manager after HTTP close,
  throwing shutdown listener, throwing closed listener, and the
  fast-drain happy path that clears the timer cleanly.
This commit is contained in:
hermes
2026-08-12 03:59:11 -07:00
parent 5b74536472
commit 432e9635bc
2 changed files with 128 additions and 22 deletions
@@ -150,6 +150,45 @@ describe('ShutdownCoordinator (DC-067)', () => {
expect(handler).toHaveBeenCalledWith('SIGTERM');
});
test('swallows exceptions thrown by shutdown event listeners', () => {
const log = makeFakeLog();
const server = makeFakeServer();
const c = createShutdownCoordinator({
server,
log,
managers: [],
});
c.on('shutdown', () => { throw new Error('listener boom'); });
// shutdown() must NOT propagate the exception — that would abort
// the entire shutdown sequence before server.close is even called.
expect(() => c.shutdown('SIGTERM')).not.toThrow();
expect(log.error).toHaveBeenCalledWith(
'shutdown',
"event listener for 'shutdown' threw",
expect.objectContaining({ error: 'listener boom' }),
);
// server.close should still have been called.
expect(server.close).toHaveBeenCalledTimes(1);
});
test('swallows exceptions thrown by closed event listeners', async () => {
const log = makeFakeLog();
const server = makeFakeServer();
const c = createShutdownCoordinator({
server,
log,
managers: [],
});
c.on('closed', () => { throw new Error('closed listener boom'); });
// process.exit is mocked; we just verify the throw doesn't bubble.
c.shutdown('SIGTERM');
await new Promise((resolve) => setImmediate(resolve));
// The closed listener threw but the exit still got recorded.
expect(exitCalls).toEqual([0]);
});
test('calls server.close() once', () => {
const server = makeFakeServer();
const c = createShutdownCoordinator({
@@ -289,36 +328,77 @@ describe('ShutdownCoordinator (DC-067)', () => {
expect(exitCalls).toEqual([0]);
expect(log.warn).toHaveBeenCalledWith(
'shutdown',
'drain timeout (1000ms) reached, force-exiting',
expect.stringContaining('drain timeout (1000ms) reached before HTTP server closed'),
);
} finally {
jest.useRealTimers();
}
});
test('clears force-exit timer when server.close fires first', () => {
test('force-exits after drainTimeoutMs if manager.stop() hangs after HTTP close', () => {
jest.useFakeTimers();
try {
const server = makeFakeServer(); // calls back on the same tick
const server = makeFakeServer(); // calls back immediately
const log = makeFakeLog();
// Manager that NEVER resolves — simulates a hung cleanup.
const hungManager = {
name: 'hung',
stop: jest.fn(() => new Promise(() => {})), // never resolves
};
const c = createShutdownCoordinator({
server,
log,
drainTimeoutMs: 1000,
managers: [],
managers: [hungManager],
});
c.shutdown('SIGTERM');
// The close callback is async (it awaits _stopManagersInOrder),
// so the synchronous c.shutdown() returns BEFORE process.exit(0)
// is recorded. Flush pending timers + microtasks.
jest.runAllTimers();
// Flush microtasks queued by the async close callback.
// After the synchronous shutdown() call: server.close has fired
// (serverClosed=true), but hungManager.stop() has been called and
// its promise is pending. managersStopped is still false.
// process.exit should NOT have been called yet.
expect(exitCalls).toEqual([]);
jest.advanceTimersByTime(1001);
// Now the safety-net timer fires — force-exit because manager hung.
expect(exitCalls).toEqual([0]);
expect(log.warn).toHaveBeenCalledWith(
'shutdown',
expect.stringContaining('after HTTP close (manager hung)'),
);
} finally {
jest.useRealTimers();
}
});
test('clears force-exit timer when manager drain completes promptly', () => {
jest.useFakeTimers();
try {
const server = makeFakeServer(); // calls back on the same tick
const log = makeFakeLog();
// Quick-stopping manager. The close callback awaits stop(),
// which resolves immediately, so managersStopped flips true
// and the safety-net timer is cleared before it can fire.
const fastManager = {
name: 'fast',
stop: jest.fn(() => Promise.resolve()),
};
const c = createShutdownCoordinator({
server,
log,
drainTimeoutMs: 1000,
managers: [fastManager],
});
c.shutdown('SIGTERM');
// Flush microtasks so the close callback's await stop() resolves,
// managersStopped flips true, the timer is cleared, and
// process.exit(0) is recorded exactly once.
return Promise.resolve().then(() => Promise.resolve()).then(() => {
expect(exitCalls).toEqual([0]);
// Advance well past the drain timeout — no extra exit should fire
// because the close callback cleared the force-exit timer.
// Advance well past the drain timeout — no extra exit should fire.
jest.advanceTimersByTime(5000);
expect(exitCalls).toEqual([0]);
});