DC-067: fix shutdown sequencing — stop managers AFTER server.close drains
Codex grade D flagged two real defects: 1. Managers were stopped before HTTP server finished draining, so in-flight requests could fail when their backing services were already down. 2. _stopManager() promises weren't awaited, contradicting the documented 'declaration order' claim for async stop methods. Fix: server.close callback now awaits _stopManagersInOrder() before exiting. The 'shutdown' event fires first (so listeners can observe the signal); the 'closed' event fires after all managers are stopped.
This commit is contained in:
@@ -54,6 +54,22 @@ class ShutdownCoordinator extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop each manager sequentially in declaration order. Each manager's
|
||||
* stop() is awaited so that a downstream manager is not stopped until
|
||||
* its upstream dependency has finished draining.
|
||||
*
|
||||
* IMPORTANT: this runs AFTER server.close() returns (see shutdown()).
|
||||
* We must wait for in-flight HTTP requests to complete before tearing
|
||||
* down the services that serve them — otherwise those requests fail
|
||||
* mid-drain with "service not found" / "monitor not running" errors.
|
||||
*/
|
||||
async _stopManagersInOrder() {
|
||||
for (const m of this.managers) {
|
||||
await this._stopManager(m);
|
||||
}
|
||||
}
|
||||
|
||||
shutdown(signal) {
|
||||
if (this._shuttingDown) {
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
|
||||
@@ -62,28 +78,26 @@ class ShutdownCoordinator extends EventEmitter {
|
||||
this._shuttingDown = true;
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
|
||||
|
||||
// Emit 'shutdown' event first so any listeners can flush their state
|
||||
// before the managers start stopping.
|
||||
// Emit 'shutdown' event first so any listeners can observe the signal
|
||||
// before the drain begins. NOTE: listeners should NOT tear down their
|
||||
// state here — that happens in the 'closed' event after server.close.
|
||||
this.emit('shutdown', signal);
|
||||
|
||||
// Stop each manager in order. Fire-and-forget — each manager's stop()
|
||||
// is expected to be fast (cancel timers, flush buffers). If a manager
|
||||
// has long async work, it should expose its own drain mechanism.
|
||||
for (const m of this.managers) {
|
||||
this._stopManager(m);
|
||||
}
|
||||
|
||||
// Close the HTTP server. Stops accepting new connections, waits for
|
||||
// in-flight requests to complete naturally.
|
||||
// Close the HTTP server FIRST. Stops accepting new connections, waits
|
||||
// for in-flight requests to complete naturally. Only AFTER close fires
|
||||
// do we tear down managers — otherwise in-flight requests could fail
|
||||
// when the services they call have already been stopped.
|
||||
let closed = false;
|
||||
try {
|
||||
this.server.close(() => {
|
||||
this.server.close(async () => {
|
||||
closed = true;
|
||||
this.log.info('shutdown', 'HTTP server closed cleanly');
|
||||
if (this._forceTimer) {
|
||||
clearTimeout(this._forceTimer);
|
||||
this._forceTimer = null;
|
||||
}
|
||||
// Now that in-flight requests are done, stop managers in order.
|
||||
await this._stopManagersInOrder();
|
||||
this.emit('closed', signal);
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -91,10 +105,14 @@ class ShutdownCoordinator extends EventEmitter {
|
||||
this.log.error('shutdown', 'server.close threw', { error: err.message });
|
||||
}
|
||||
|
||||
// Force-exit if drain doesn't complete in time.
|
||||
// Force-exit if drain doesn't complete in time. This is the safety
|
||||
// net for the case where a long-polling request or a stuck handler
|
||||
// never returns and the drain never finishes.
|
||||
this._forceTimer = setTimeout(() => {
|
||||
if (closed) return; // race: server.close fired AND timer fired
|
||||
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`);
|
||||
// On force-exit we don't run manager.stop() — the process is
|
||||
// about to die anyway, and SIGKILL will be next if we don't exit.
|
||||
process.exit(0);
|
||||
}, this.drainTimeoutMs);
|
||||
if (this._forceTimer && typeof this._forceTimer.unref === 'function') {
|
||||
|
||||
Reference in New Issue
Block a user