[grade=B urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq] DC-134/135/136: Shipdeck integration - data-driven login pages, deploy events, badge suppression

DC-134: /api/v1/auth/login-page serves a generic gated auto-login page for
any service registered in services.json without a curated flow (App Selector
installs, DC-131 git installs). Curated pages always win; unknown services
still 404; sanitizer keeps digits/hyphens (shipdeck-style ids); display
names HTML-escaped. Kills the sso-gate.js edit + restart per new install.

DC-135: shipdeck journal.jsonl tail worker (startShipdeckWorker) ingests
deploy/rollback lifecycle rows into the Security Center as
source_type=shipdeck (notice/success, error/failure via verify[] block).

DC-136: deploy-aware badge suppression - suppressDuringDeploy() before the
bridge call in /deploy and /rollback, clearDeploySuppression() in finally
(every exit path incl. rejected fetches), reference-counted for overlapping
deploys, 10-min TTL auto-expiry (HEALTH_DEPLOY_SUPPRESS_MAX_MS).

23 new tests across 4 suites; full suite 2923/2923 green.

Codex judge: C (r1) -> C (r2) -> B (r3) -> B zero-blockers (r4,
urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq).
This commit is contained in:
DashCaddy Polish Loop
2026-09-16 03:10:54 -07:00
parent 0dd8493f98
commit 11c719e635
10 changed files with 912 additions and 11 deletions
+1
View File
@@ -786,6 +786,7 @@ async function createApp() {
log: ctx.log,
auditLogger: ctx.auditLogger,
fetchT: ctx.fetchT,
healthChecker, // DC-136: deploy-aware badge suppression
}));
// Log Insights — plain English activity summary + safe log disposal
+73 -1
View File
@@ -56,6 +56,10 @@ function readPositiveIntEnv(name, fallback) {
const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2);
const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1);
// DC-136: max time a deploy-suppression flag may hold a badge. A shipdeck
// deploy blackholes probes for seconds to a couple of minutes; the flag
// auto-expires so a crashed deploy can never silence a badge forever.
const DEPLOY_SUPPRESS_MAX_MS = readPositiveIntEnv('HEALTH_DEPLOY_SUPPRESS_MAX_MS', 10 * 60 * 1000);
class HealthChecker extends EventEmitter {
constructor() {
@@ -84,6 +88,59 @@ class HealthChecker extends EventEmitter {
// HIGHER generation than the captured one marks the capture as stale. Entry
// is deleted when the service is removed, so the live map cannot leak.
this.removedGenerations = new Map();
// DC-136: serviceId -> expires-at (ms epoch). While active and unexpired,
// probes that land during a shipdeck deploy/rollback are recorded in raw
// history but don't drive the displayed badge or incident transitions.
this.deploySuppressedUntil = new Map();
// DC-136 r3 (judge polish): active-suppression reference counts so
// overlapping deploy requests for the same service can't clear each
// other's window while one of them is still in flight.
this.deploySuppressRefs = new Map();
}
/**
* DC-136: mark a service as mid-deploy. While suppressed, probe results
* still land in history (full fidelity preserved) but do NOT flip the
* displayed badge or open/resolve outage incidents — a service that is
* momentarily blackholed by its own redeploy should not page anyone.
* Suppression auto-expires after HEALTH_DEPLOY_SUPPRESS_MAX_MS (10 min)
* so a crashed deploy cannot silence a badge forever.
*/
suppressDuringDeploy(serviceId, ttlMs) {
const ttl = Number.isSafeInteger(ttlMs) && ttlMs > 0 ? ttlMs : DEPLOY_SUPPRESS_MAX_MS;
this.deploySuppressedUntil.set(serviceId, Date.now() + Math.min(ttl, DEPLOY_SUPPRESS_MAX_MS));
this.deploySuppressRefs.set(serviceId, (this.deploySuppressRefs.get(serviceId) || 0) + 1);
}
/**
* DC-136 (judge round 2): end the suppression window explicitly. The
* deploys routes call this when the bridge operation COMPLETES — on
* success because shipdeck health-verified the service before returning,
* and on failure because real downtime must become visible immediately
* (a failed deploy must never start a fresh 10-minute silence window).
* Reference-counted (judge round 3): with overlapping deploys of the
* same service, the window survives until the LAST in-flight request
* finishes. Always invoked from a `finally` so a thrown bridge error
* can never leak an active suppression window.
*/
clearDeploySuppression(serviceId) {
const refs = (this.deploySuppressRefs.get(serviceId) || 0) - 1;
if (refs > 0) {
this.deploySuppressRefs.set(serviceId, refs);
return;
}
this.deploySuppressRefs.delete(serviceId);
this.deploySuppressedUntil.delete(serviceId);
}
_isDeploySuppressed(serviceId) {
const until = this.deploySuppressedUntil.get(serviceId);
if (until === undefined) return false;
if (Date.now() >= until) {
this.deploySuppressedUntil.delete(serviceId);
return false;
}
return true;
}
/**
@@ -432,6 +489,15 @@ class HealthChecker extends EventEmitter {
// _computeDisplayedStatus compares the raw probe against the DISPLAYED
// status (not the previous raw status), so the "consecutive since
// change" counter doesn't depend on the order of writes here.
// DC-136: during a shipdeck deploy/rollback window the probe result is
// still recorded (history + currentStatus above stay full-fidelity) but
// must not drive the badge — a redeploy blackholes the service for
// seconds and the red flip would be pure deploy noise. The displayed
// map is left untouched; the badge simply holds its pre-deploy state.
if (this._isDeploySuppressed(serviceId)) {
return;
}
const displayed = this._computeDisplayedStatus(serviceId, status);
const previousDisplayed = this.displayedStatus.get(serviceId);
const displayChanged =
@@ -456,7 +522,13 @@ class HealthChecker extends EventEmitter {
* Check for incidents (downtime, slow response, etc.)
*/
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
// DC-136: probe results inside a deploy-suppression window are deploy
// noise by definition (timeouts, 5xx from a restarting unit, huge
// response times) — they must not open outage/slow-response incidents.
// Slow-response and SLA checks are included in the skip; SLA math runs
// on history uptime which is unaffected by this early return.
if (this._isDeploySuppressed(serviceId)) return;
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
// the same signal that flips the dashboard badge. A single raw "down"
// blip that hysteresis suppresses must not open a critical outage
+1 -1
View File
@@ -40,7 +40,7 @@ const TRIM_TARGET_FACTOR = 0.8; // post-trim target: ≤80% of the byte budget
const DEFAULT_TRIM_SIZE_LIMIT = parseInt(
process.env.SECURITY_EVENT_TRIM_BYTES || String(50 * 1024 * 1024), 10);
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent']);
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent', 'shipdeck']);
const VALID_SEVERITIES = new Set(['info', 'notice', 'warn', 'error', 'critical']);
const VALID_OUTCOMES = new Set(['success', 'failure', 'denied', 'blocked', 'rate-limited', 'error', 'unknown']);
@@ -15,6 +15,11 @@
* 3. fail2ban log tail — parses /var/log/fail2ban.log for ban/unban
* actions. SSH jail is the default; can extend to other jails.
*
* 4. shipdeck journal tail — parses /var/lib/shipdeck/journal.jsonl
* (JSONL, one row per lifecycle event, DC-135). Deploy/rollback rows
* become source_type='shipdeck' security events — an unexplained
* redeploy is a security-relevant event.
*
* Each worker:
* - Starts on app boot (via server.js)
* - Tracks its byte offset in the log file so it survives restarts (no re-emit)
@@ -422,6 +427,82 @@ function startFail2banWorker({ log } = {}) {
});
}
/**
* DC-135: shipdeck journal tail worker. The shipdeck CLI appends one JSON
* row per deploy/rollback lifecycle event to /var/lib/shipdeck/journal.jsonl
* (format documented in the shipdeck repo: time, service, host, epoch,
* action, duration_s, spec, verify[]). Tail it so deploys appear in the
* Security Center timeline next to auth and perimeter events — an
* unexplained redeploy IS a security-relevant event. We ingest only
* lifecycle actions (deploy/rollback), not health probes, so the store
* isn't flooded by routine checks.
*
* Severity mapping:
* deploy/rollback success -> notice (infrastructure changed)
* deploy/rollback failure -> error
* unknown/unexpected action -> notice
*/
function startShipdeckWorker({ log: logger = log } = {}) {
const journalPath = process.env.SHIPDECK_JOURNAL_FILE || '/var/lib/shipdeck/journal.jsonl';
const stateFile = path.join(platformPaths.dataDir, '.shipdeck-tail-offset');
const store = getStore({ log: logger });
const LIFECYCLE_ACTIONS = new Set(['deploy', 'rollback']);
let missingWarned = false;
function warnIfMissing() {
if (missingWarned) return;
fs.stat(journalPath, (err) => {
if (!err) return;
missingWarned = true;
logger.warn?.('events', `shipdeck journal not found at ${journalPath} — shipdeck-source security events disabled (set SHIPDECK_JOURNAL_FILE)`, { worker: 'shipdeck' });
});
}
warnIfMissing();
return createTail({
filePath: journalPath,
stateFile,
label: 'shipdeck',
firstStartMaxBytes: 1 * 1024 * 1024,
onAppear: () => {
logger.info?.('events', `shipdeck journal active at ${journalPath} — shipdeck-source security events enabled`, { worker: 'shipdeck' });
},
onLine: (line) => {
let row;
try { row = JSON.parse(line); }
catch { return; } // journal is JSONL; skip torn/unparseable lines
if (!row || typeof row !== 'object') return;
const action = typeof row.action === 'string' ? row.action : 'unknown';
// Health/verify/lifecycle-noise rows are not security events.
if (!LIFECYCLE_ACTIONS.has(action)) return;
const okAll = Array.isArray(row.verify) && row.verify.length > 0
? row.verify.every(v => v && v.ok === true)
: true; // no verify block -> treat as accepted (local_mode rows)
const failed = okAll === false || (typeof row.error === 'string' && row.error.length > 0);
store.append({
source_host: typeof row.host === 'string' && row.host ? row.host : HOSTNAME,
source_type: 'shipdeck',
actor: null,
target: typeof row.service === 'string' ? row.service : null,
action: `shipdeck.${action}`,
outcome: failed ? 'error' : 'success',
severity: failed ? 'error' : 'notice',
message: failed
? `shipdeck ${action} of ${row.service} FAILED (epoch ${row.epoch})`
: `shipdeck ${action} of ${row.service} succeeded (epoch ${row.epoch}, ${(row.duration_s || 0).toFixed ? row.duration_s.toFixed(1) : row.duration_s}s)`,
metadata: {
epoch: row.epoch || null,
duration_s: row.duration_s || null,
record: (row.spec && row.spec.record) || null,
verify_http: (row.spec && row.spec.verify_http) || null,
verify: Array.isArray(row.verify) ? row.verify : null,
},
});
},
});
}
/**
* Start all workers. Returns a stop function that shuts them all down.
*/
@@ -433,6 +514,8 @@ function startAll({ log } = {}) {
catch (e) { log.error('events', e, { worker: 'shared_bans', phase: 'start' }); }
try { workers.push(startFail2banWorker({ log })); }
catch (e) { log.error('events', e, { worker: 'fail2ban', phase: 'start' }); }
try { workers.push(startShipdeckWorker({ log })); }
catch (e) { log.error('events', e, { worker: 'shipdeck', phase: 'start' }); }
return {
stop() { workers.forEach(w => { try { w.stop(); } catch {} }); },
workers,
@@ -444,6 +527,7 @@ module.exports = {
startCaddyWorker,
startSharedBansWorker,
startFail2banWorker,
startShipdeckWorker,
startAll,
resolveCaddyAction,
};