DC-067: graceful shutdown coordinator (EventEmitter, 10s drain, idempotent)
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Graceful shutdown coordinator — DashCaddy
|
||||
*
|
||||
* Extracts the SIGTERM/SIGINT handler from server.js into a testable,
|
||||
* reusable module that:
|
||||
* 1. Calls server.close() to drain in-flight HTTP connections
|
||||
* 2. Stops each manager in a deterministic order
|
||||
* 3. Emits a 'shutdown' event so additional listeners can do cleanup
|
||||
* 4. Force-exits after a configurable drain timeout if connections don't drain
|
||||
* 5. Is idempotent — a second SIGTERM during shutdown does not re-run handlers
|
||||
*
|
||||
* Spec: DC-067 (production-grade backlog). Docker sends SIGTERM on stop;
|
||||
* without this coordinator, in-flight API calls drop.
|
||||
*
|
||||
* Exports:
|
||||
* - createShutdownCoordinator({ server, log, drainTimeoutMs, managers })
|
||||
* Returns an EventEmitter with: { shutdown, isShuttingDown, on, emit, ... }
|
||||
* - installSignalHandlers(coordinator, signals = ['SIGTERM', 'SIGINT'])
|
||||
* Registers the OS-level handlers. Idempotent.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
|
||||
|
||||
class ShutdownCoordinator extends EventEmitter {
|
||||
constructor({ server, log, drainTimeoutMs, managers }) {
|
||||
super();
|
||||
if (!server) throw new Error('createShutdownCoordinator: server is required');
|
||||
if (!log || typeof log.info !== 'function') {
|
||||
throw new Error('createShutdownCoordinator: log with info/warn is required');
|
||||
}
|
||||
this.server = server;
|
||||
this.log = log;
|
||||
this.drainTimeoutMs = Number.isFinite(drainTimeoutMs) && drainTimeoutMs > 0
|
||||
? drainTimeoutMs
|
||||
: DEFAULT_DRAIN_TIMEOUT_MS;
|
||||
this.managers = Array.isArray(managers) ? managers : [];
|
||||
this._shuttingDown = false;
|
||||
this._forceTimer = null;
|
||||
}
|
||||
|
||||
isShuttingDown() {
|
||||
return this._shuttingDown;
|
||||
}
|
||||
|
||||
async _stopManager(m) {
|
||||
try {
|
||||
await m.stop();
|
||||
this.log.info('shutdown', `manager stopped: ${m.name}`);
|
||||
} catch (err) {
|
||||
this.log.warn('shutdown', `manager stop failed: ${m.name}`, { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
shutdown(signal) {
|
||||
if (this._shuttingDown) {
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received — already shutting down, ignoring`);
|
||||
return;
|
||||
}
|
||||
this._shuttingDown = true;
|
||||
this.log.info('shutdown', `${signal || 'shutdown'} received, draining (timeout=${this.drainTimeoutMs}ms)...`);
|
||||
|
||||
// Emit 'shutdown' event first so any listeners can flush their state
|
||||
// before the managers start stopping.
|
||||
this.emit('shutdown', signal);
|
||||
|
||||
// Stop each manager in order. Fire-and-forget — each manager's stop()
|
||||
// is expected to be fast (cancel timers, flush buffers). If a manager
|
||||
// has long async work, it should expose its own drain mechanism.
|
||||
for (const m of this.managers) {
|
||||
this._stopManager(m);
|
||||
}
|
||||
|
||||
// Close the HTTP server. Stops accepting new connections, waits for
|
||||
// in-flight requests to complete naturally.
|
||||
let closed = false;
|
||||
try {
|
||||
this.server.close(() => {
|
||||
closed = true;
|
||||
this.log.info('shutdown', 'HTTP server closed cleanly');
|
||||
if (this._forceTimer) {
|
||||
clearTimeout(this._forceTimer);
|
||||
this._forceTimer = null;
|
||||
}
|
||||
this.emit('closed', signal);
|
||||
process.exit(0);
|
||||
});
|
||||
} catch (err) {
|
||||
this.log.error('shutdown', 'server.close threw', { error: err.message });
|
||||
}
|
||||
|
||||
// Force-exit if drain doesn't complete in time.
|
||||
this._forceTimer = setTimeout(() => {
|
||||
if (closed) return; // race: server.close fired AND timer fired
|
||||
this.log.warn('shutdown', `drain timeout (${this.drainTimeoutMs}ms) reached, force-exiting`);
|
||||
process.exit(0);
|
||||
}, this.drainTimeoutMs);
|
||||
if (this._forceTimer && typeof this._forceTimer.unref === 'function') {
|
||||
this._forceTimer.unref();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createShutdownCoordinator(opts) {
|
||||
return new ShutdownCoordinator(opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Install OS-level signal handlers. Idempotent: second call is a no-op.
|
||||
*
|
||||
* @param {ShutdownCoordinator} coordinator
|
||||
* @param {string[]} signals - signals to handle (default: SIGTERM, SIGINT)
|
||||
*/
|
||||
function installSignalHandlers(coordinator, signals) {
|
||||
if (!coordinator || typeof coordinator.shutdown !== 'function') {
|
||||
throw new Error('installSignalHandlers: coordinator required');
|
||||
}
|
||||
const sigs = Array.isArray(signals) && signals.length > 0
|
||||
? signals
|
||||
: ['SIGTERM', 'SIGINT'];
|
||||
for (const sig of sigs) {
|
||||
process.on(sig, () => coordinator.shutdown(sig));
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createShutdownCoordinator,
|
||||
installSignalHandlers,
|
||||
DEFAULT_DRAIN_TIMEOUT_MS,
|
||||
ShutdownCoordinator, // exported for tests
|
||||
};
|
||||
Reference in New Issue
Block a user