[grade=B urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq] DC-134/135/136: Shipdeck integration - data-driven login pages, deploy events, badge suppression
DC-134: /api/v1/auth/login-page serves a generic gated auto-login page for any service registered in services.json without a curated flow (App Selector installs, DC-131 git installs). Curated pages always win; unknown services still 404; sanitizer keeps digits/hyphens (shipdeck-style ids); display names HTML-escaped. Kills the sso-gate.js edit + restart per new install. DC-135: shipdeck journal.jsonl tail worker (startShipdeckWorker) ingests deploy/rollback lifecycle rows into the Security Center as source_type=shipdeck (notice/success, error/failure via verify[] block). DC-136: deploy-aware badge suppression - suppressDuringDeploy() before the bridge call in /deploy and /rollback, clearDeploySuppression() in finally (every exit path incl. rejected fetches), reference-counted for overlapping deploys, 10-min TTL auto-expiry (HEALTH_DEPLOY_SUPPRESS_MAX_MS). 23 new tests across 4 suites; full suite 2923/2923 green. Codex judge: C (r1) -> C (r2) -> B (r3) -> B zero-blockers (r4, urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq).
This commit is contained in:
@@ -786,6 +786,7 @@ async function createApp() {
|
||||
log: ctx.log,
|
||||
auditLogger: ctx.auditLogger,
|
||||
fetchT: ctx.fetchT,
|
||||
healthChecker, // DC-136: deploy-aware badge suppression
|
||||
}));
|
||||
|
||||
// Log Insights — plain English activity summary + safe log disposal
|
||||
|
||||
@@ -56,6 +56,10 @@ function readPositiveIntEnv(name, fallback) {
|
||||
|
||||
const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2);
|
||||
const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1);
|
||||
// DC-136: max time a deploy-suppression flag may hold a badge. A shipdeck
|
||||
// deploy blackholes probes for seconds to a couple of minutes; the flag
|
||||
// auto-expires so a crashed deploy can never silence a badge forever.
|
||||
const DEPLOY_SUPPRESS_MAX_MS = readPositiveIntEnv('HEALTH_DEPLOY_SUPPRESS_MAX_MS', 10 * 60 * 1000);
|
||||
|
||||
class HealthChecker extends EventEmitter {
|
||||
constructor() {
|
||||
@@ -84,6 +88,59 @@ class HealthChecker extends EventEmitter {
|
||||
// HIGHER generation than the captured one marks the capture as stale. Entry
|
||||
// is deleted when the service is removed, so the live map cannot leak.
|
||||
this.removedGenerations = new Map();
|
||||
// DC-136: serviceId -> expires-at (ms epoch). While active and unexpired,
|
||||
// probes that land during a shipdeck deploy/rollback are recorded in raw
|
||||
// history but don't drive the displayed badge or incident transitions.
|
||||
this.deploySuppressedUntil = new Map();
|
||||
// DC-136 r3 (judge polish): active-suppression reference counts so
|
||||
// overlapping deploy requests for the same service can't clear each
|
||||
// other's window while one of them is still in flight.
|
||||
this.deploySuppressRefs = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* DC-136: mark a service as mid-deploy. While suppressed, probe results
|
||||
* still land in history (full fidelity preserved) but do NOT flip the
|
||||
* displayed badge or open/resolve outage incidents — a service that is
|
||||
* momentarily blackholed by its own redeploy should not page anyone.
|
||||
* Suppression auto-expires after HEALTH_DEPLOY_SUPPRESS_MAX_MS (10 min)
|
||||
* so a crashed deploy cannot silence a badge forever.
|
||||
*/
|
||||
suppressDuringDeploy(serviceId, ttlMs) {
|
||||
const ttl = Number.isSafeInteger(ttlMs) && ttlMs > 0 ? ttlMs : DEPLOY_SUPPRESS_MAX_MS;
|
||||
this.deploySuppressedUntil.set(serviceId, Date.now() + Math.min(ttl, DEPLOY_SUPPRESS_MAX_MS));
|
||||
this.deploySuppressRefs.set(serviceId, (this.deploySuppressRefs.get(serviceId) || 0) + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* DC-136 (judge round 2): end the suppression window explicitly. The
|
||||
* deploys routes call this when the bridge operation COMPLETES — on
|
||||
* success because shipdeck health-verified the service before returning,
|
||||
* and on failure because real downtime must become visible immediately
|
||||
* (a failed deploy must never start a fresh 10-minute silence window).
|
||||
* Reference-counted (judge round 3): with overlapping deploys of the
|
||||
* same service, the window survives until the LAST in-flight request
|
||||
* finishes. Always invoked from a `finally` so a thrown bridge error
|
||||
* can never leak an active suppression window.
|
||||
*/
|
||||
clearDeploySuppression(serviceId) {
|
||||
const refs = (this.deploySuppressRefs.get(serviceId) || 0) - 1;
|
||||
if (refs > 0) {
|
||||
this.deploySuppressRefs.set(serviceId, refs);
|
||||
return;
|
||||
}
|
||||
this.deploySuppressRefs.delete(serviceId);
|
||||
this.deploySuppressedUntil.delete(serviceId);
|
||||
}
|
||||
|
||||
_isDeploySuppressed(serviceId) {
|
||||
const until = this.deploySuppressedUntil.get(serviceId);
|
||||
if (until === undefined) return false;
|
||||
if (Date.now() >= until) {
|
||||
this.deploySuppressedUntil.delete(serviceId);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -432,6 +489,15 @@ class HealthChecker extends EventEmitter {
|
||||
// _computeDisplayedStatus compares the raw probe against the DISPLAYED
|
||||
// status (not the previous raw status), so the "consecutive since
|
||||
// change" counter doesn't depend on the order of writes here.
|
||||
// DC-136: during a shipdeck deploy/rollback window the probe result is
|
||||
// still recorded (history + currentStatus above stay full-fidelity) but
|
||||
// must not drive the badge — a redeploy blackholes the service for
|
||||
// seconds and the red flip would be pure deploy noise. The displayed
|
||||
// map is left untouched; the badge simply holds its pre-deploy state.
|
||||
if (this._isDeploySuppressed(serviceId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const displayed = this._computeDisplayedStatus(serviceId, status);
|
||||
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||
const displayChanged =
|
||||
@@ -456,7 +522,13 @@ class HealthChecker extends EventEmitter {
|
||||
* Check for incidents (downtime, slow response, etc.)
|
||||
*/
|
||||
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
||||
|
||||
// DC-136: probe results inside a deploy-suppression window are deploy
|
||||
// noise by definition (timeouts, 5xx from a restarting unit, huge
|
||||
// response times) — they must not open outage/slow-response incidents.
|
||||
// Slow-response and SLA checks are included in the skip; SLA math runs
|
||||
// on history uptime which is unaffected by this early return.
|
||||
if (this._isDeploySuppressed(serviceId)) return;
|
||||
|
||||
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
||||
// the same signal that flips the dashboard badge. A single raw "down"
|
||||
// blip that hysteresis suppresses must not open a critical outage
|
||||
|
||||
@@ -40,7 +40,7 @@ const TRIM_TARGET_FACTOR = 0.8; // post-trim target: ≤80% of the byte budget
|
||||
const DEFAULT_TRIM_SIZE_LIMIT = parseInt(
|
||||
process.env.SECURITY_EVENT_TRIM_BYTES || String(50 * 1024 * 1024), 10);
|
||||
|
||||
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent']);
|
||||
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent', 'shipdeck']);
|
||||
const VALID_SEVERITIES = new Set(['info', 'notice', 'warn', 'error', 'critical']);
|
||||
const VALID_OUTCOMES = new Set(['success', 'failure', 'denied', 'blocked', 'rate-limited', 'error', 'unknown']);
|
||||
|
||||
|
||||
@@ -15,6 +15,11 @@
|
||||
* 3. fail2ban log tail — parses /var/log/fail2ban.log for ban/unban
|
||||
* actions. SSH jail is the default; can extend to other jails.
|
||||
*
|
||||
* 4. shipdeck journal tail — parses /var/lib/shipdeck/journal.jsonl
|
||||
* (JSONL, one row per lifecycle event, DC-135). Deploy/rollback rows
|
||||
* become source_type='shipdeck' security events — an unexplained
|
||||
* redeploy is a security-relevant event.
|
||||
*
|
||||
* Each worker:
|
||||
* - Starts on app boot (via server.js)
|
||||
* - Tracks its byte offset in the log file so it survives restarts (no re-emit)
|
||||
@@ -422,6 +427,82 @@ function startFail2banWorker({ log } = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* DC-135: shipdeck journal tail worker. The shipdeck CLI appends one JSON
|
||||
* row per deploy/rollback lifecycle event to /var/lib/shipdeck/journal.jsonl
|
||||
* (format documented in the shipdeck repo: time, service, host, epoch,
|
||||
* action, duration_s, spec, verify[]). Tail it so deploys appear in the
|
||||
* Security Center timeline next to auth and perimeter events — an
|
||||
* unexplained redeploy IS a security-relevant event. We ingest only
|
||||
* lifecycle actions (deploy/rollback), not health probes, so the store
|
||||
* isn't flooded by routine checks.
|
||||
*
|
||||
* Severity mapping:
|
||||
* deploy/rollback success -> notice (infrastructure changed)
|
||||
* deploy/rollback failure -> error
|
||||
* unknown/unexpected action -> notice
|
||||
*/
|
||||
function startShipdeckWorker({ log: logger = log } = {}) {
|
||||
const journalPath = process.env.SHIPDECK_JOURNAL_FILE || '/var/lib/shipdeck/journal.jsonl';
|
||||
const stateFile = path.join(platformPaths.dataDir, '.shipdeck-tail-offset');
|
||||
const store = getStore({ log: logger });
|
||||
|
||||
const LIFECYCLE_ACTIONS = new Set(['deploy', 'rollback']);
|
||||
|
||||
let missingWarned = false;
|
||||
function warnIfMissing() {
|
||||
if (missingWarned) return;
|
||||
fs.stat(journalPath, (err) => {
|
||||
if (!err) return;
|
||||
missingWarned = true;
|
||||
logger.warn?.('events', `shipdeck journal not found at ${journalPath} — shipdeck-source security events disabled (set SHIPDECK_JOURNAL_FILE)`, { worker: 'shipdeck' });
|
||||
});
|
||||
}
|
||||
warnIfMissing();
|
||||
|
||||
return createTail({
|
||||
filePath: journalPath,
|
||||
stateFile,
|
||||
label: 'shipdeck',
|
||||
firstStartMaxBytes: 1 * 1024 * 1024,
|
||||
onAppear: () => {
|
||||
logger.info?.('events', `shipdeck journal active at ${journalPath} — shipdeck-source security events enabled`, { worker: 'shipdeck' });
|
||||
},
|
||||
onLine: (line) => {
|
||||
let row;
|
||||
try { row = JSON.parse(line); }
|
||||
catch { return; } // journal is JSONL; skip torn/unparseable lines
|
||||
if (!row || typeof row !== 'object') return;
|
||||
const action = typeof row.action === 'string' ? row.action : 'unknown';
|
||||
// Health/verify/lifecycle-noise rows are not security events.
|
||||
if (!LIFECYCLE_ACTIONS.has(action)) return;
|
||||
const okAll = Array.isArray(row.verify) && row.verify.length > 0
|
||||
? row.verify.every(v => v && v.ok === true)
|
||||
: true; // no verify block -> treat as accepted (local_mode rows)
|
||||
const failed = okAll === false || (typeof row.error === 'string' && row.error.length > 0);
|
||||
store.append({
|
||||
source_host: typeof row.host === 'string' && row.host ? row.host : HOSTNAME,
|
||||
source_type: 'shipdeck',
|
||||
actor: null,
|
||||
target: typeof row.service === 'string' ? row.service : null,
|
||||
action: `shipdeck.${action}`,
|
||||
outcome: failed ? 'error' : 'success',
|
||||
severity: failed ? 'error' : 'notice',
|
||||
message: failed
|
||||
? `shipdeck ${action} of ${row.service} FAILED (epoch ${row.epoch})`
|
||||
: `shipdeck ${action} of ${row.service} succeeded (epoch ${row.epoch}, ${(row.duration_s || 0).toFixed ? row.duration_s.toFixed(1) : row.duration_s}s)`,
|
||||
metadata: {
|
||||
epoch: row.epoch || null,
|
||||
duration_s: row.duration_s || null,
|
||||
record: (row.spec && row.spec.record) || null,
|
||||
verify_http: (row.spec && row.spec.verify_http) || null,
|
||||
verify: Array.isArray(row.verify) ? row.verify : null,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start all workers. Returns a stop function that shuts them all down.
|
||||
*/
|
||||
@@ -433,6 +514,8 @@ function startAll({ log } = {}) {
|
||||
catch (e) { log.error('events', e, { worker: 'shared_bans', phase: 'start' }); }
|
||||
try { workers.push(startFail2banWorker({ log })); }
|
||||
catch (e) { log.error('events', e, { worker: 'fail2ban', phase: 'start' }); }
|
||||
try { workers.push(startShipdeckWorker({ log })); }
|
||||
catch (e) { log.error('events', e, { worker: 'shipdeck', phase: 'start' }); }
|
||||
return {
|
||||
stop() { workers.forEach(w => { try { w.stop(); } catch {} }); },
|
||||
workers,
|
||||
@@ -444,6 +527,7 @@ module.exports = {
|
||||
startCaddyWorker,
|
||||
startSharedBansWorker,
|
||||
startFail2banWorker,
|
||||
startShipdeckWorker,
|
||||
startAll,
|
||||
resolveCaddyAction,
|
||||
};
|
||||
Reference in New Issue
Block a user