DC-067: fix shutdown sequencing — stop managers AFTER server.close drains

Codex grade D flagged two real defects:
1. Managers were stopped before HTTP server finished draining, so in-flight
   requests could fail when their backing services were already down.
2. _stopManager() promises weren't awaited, contradicting the documented
   'declaration order' claim for async stop methods.

Fix: server.close callback now awaits _stopManagersInOrder() before
exiting. The 'shutdown' event fires first (so listeners can observe the
signal); the 'closed' event fires after all managers are stopped.
This commit is contained in:
hermes
2026-08-12 03:46:42 -07:00
parent 88ff260e5e
commit bb01a77ae7
2 changed files with 85 additions and 31 deletions
+31 -13
View File
@@ -54,6 +54,22 @@ class ShutdownCoordinator extends EventEmitter {
}
}
/**
* Stop each manager sequentially in declaration order. Each manager's
* stop() is awaited so that a downstream manager is not stopped until
* its upstream dependency has finished draining.
*
* IMPORTANT: this runs AFTER server.close() returns (see shutdown()).
* We must wait for in-flight HTTP requests to complete before tearing
* down the services that serve them — otherwise those requests fail
* mid-drain with "service not found" / "monitor not running" errors.
*/
async _stopManagersInOrder() {
for (const m of this.managers) {
await this._stopManager(m);
}
}
shutdown(signal) {
if (this._shuttingDown) {
this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
@@ -62,28 +78,26 @@ class ShutdownCoordinator extends EventEmitter {
this._shuttingDown = true;
this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
// Emit 'shutdown' event first so any listeners can flush their state
// before the managers start stopping.
// Emit 'shutdown' event first so any listeners can observe the signal
// before the drain begins. NOTE: listeners should NOT tear down their
// state here — that happens in the 'closed' event after server.close.
this.emit('shutdown', signal);
// Stop each manager in order. Fire-and-forget — each manager's stop()
// is expected to be fast (cancel timers, flush buffers). If a manager
// has long async work, it should expose its own drain mechanism.
for (const m of this.managers) {
this._stopManager(m);
}
// Close the HTTP server. Stops accepting new connections, waits for
// in-flight requests to complete naturally.
// Close the HTTP server FIRST. Stops accepting new connections, waits
// for in-flight requests to complete naturally. Only AFTER close fires
// do we tear down managers — otherwise in-flight requests could fail
// when the services they call have already been stopped.
let closed = false;
try {
this.server.close(() => {
this.server.close(async () => {
closed = true;
this.log.info('shutdown', 'HTTP server closed cleanly');
if (this._forceTimer) {
clearTimeout(this._forceTimer);
this._forceTimer = null;
}
// Now that in-flight requests are done, stop managers in order.
await this._stopManagersInOrder();
this.emit('closed', signal);
process.exit(0);
});
@@ -91,10 +105,14 @@ class ShutdownCoordinator extends EventEmitter {
this.log.error('shutdown', 'server.close threw', { error: err.message });
}
// Force-exit if drain doesn't complete in time.
// Force-exit if drain doesn't complete in time. This is the safety
// net for the case where a long-polling request or a stuck handler
// never returns and the drain never finishes.
this._forceTimer = setTimeout(() => {
if (closed) return; // race: server.close fired AND timer fired
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`);
// On force-exit we don't run manager.stop() — the process is
// about to die anyway, and SIGKILL will be next if we don't exit.
process.exit(0);
}, this.drainTimeoutMs);
if (this._forceTimer && typeof this._forceTimer.unref === 'function') {