[grade=B urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq] DC-134/135/136: Shipdeck integration - data-driven login pages, deploy events, badge suppression
DC-134: /api/v1/auth/login-page serves a generic gated auto-login page for any service registered in services.json without a curated flow (App Selector installs, DC-131 git installs). Curated pages always win; unknown services still 404; sanitizer keeps digits/hyphens (shipdeck-style ids); display names HTML-escaped. Kills the sso-gate.js edit + restart per new install. DC-135: shipdeck journal.jsonl tail worker (startShipdeckWorker) ingests deploy/rollback lifecycle rows into the Security Center as source_type=shipdeck (notice/success, error/failure via verify[] block). DC-136: deploy-aware badge suppression - suppressDuringDeploy() before the bridge call in /deploy and /rollback, clearDeploySuppression() in finally (every exit path incl. rejected fetches), reference-counted for overlapping deploys, 10-min TTL auto-expiry (HEALTH_DEPLOY_SUPPRESS_MAX_MS). 23 new tests across 4 suites; full suite 2923/2923 green. Codex judge: C (r1) -> C (r2) -> B (r3) -> B zero-blockers (r4, urn:ump:3wvdd3yegylr7e2pzn6tebyq5bisf2awmss7p4j3uwpwsujtnwoq).
This commit is contained in:
@@ -0,0 +1,130 @@
|
|||||||
|
/**
|
||||||
|
* DC-136: deploy-aware badge suppression in the health checker.
|
||||||
|
*
|
||||||
|
* A shipdeck deploy/rollback restarts the target unit; probes that land
|
||||||
|
* during that window blackhole (timeouts / 5xx) and — before this change —
|
||||||
|
* flipped the badge red and opened outage incidents for what is routine
|
||||||
|
* deploy noise.
|
||||||
|
*
|
||||||
|
* Pins:
|
||||||
|
* - suppressDuringDeploy() holds the displayed badge through down probes
|
||||||
|
* (even past DOWN_THRESHOLD) and emits nothing.
|
||||||
|
* - Raw history keeps every probe (full fidelity preserved).
|
||||||
|
* - checkForIncidents opens no outage/slow-response incident while
|
||||||
|
* suppressed.
|
||||||
|
* - After expiry the checker behaves exactly as before (down probes flip
|
||||||
|
* the badge again).
|
||||||
|
* - TTL is clamped to HEALTH_DEPLOY_SUPPRESS_MAX_MS.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
|
||||||
|
const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc136-deploysuppress-'));
|
||||||
|
process.env.HEALTH_DATA_DIR = TMP_DIR;
|
||||||
|
process.env.HEALTH_CONFIG_FILE = path.join(TMP_DIR, 'health-config.json');
|
||||||
|
process.env.HEALTH_HISTORY_FILE = path.join(TMP_DIR, 'health-history.json');
|
||||||
|
process.env.HEALTH_DEPLOY_SUPPRESS_MAX_MS = '60000'; // test-visible clamp ceiling
|
||||||
|
|
||||||
|
// Module exports a singleton instance — same pattern as
|
||||||
|
// health-checker-hysteresis.test.js.
|
||||||
|
const healthCheckerSingleton = require('../src/monitoring/health-checker');
|
||||||
|
|
||||||
|
function makeUp(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'up',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 200,
|
||||||
|
message: 'Service is healthy',
|
||||||
|
details: { headers: {}, bodyLength: 12 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDown(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'down',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 500,
|
||||||
|
message: 'fail',
|
||||||
|
details: { headers: {}, bodyLength: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DC-136: deploy suppression on the dashboard badge', () => {
|
||||||
|
let hc;
|
||||||
|
let emitSpy;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
hc = healthCheckerSingleton;
|
||||||
|
hc.displayedStatus = new Map();
|
||||||
|
hc.consecutiveSinceChange = new Map();
|
||||||
|
hc.currentStatus = new Map();
|
||||||
|
hc.history = {};
|
||||||
|
hc.deploySuppressedUntil = new Map();
|
||||||
|
hc.deploySuppressRefs = new Map(); // judge r4 polish: reset ref counts too
|
||||||
|
hc.incidents = [];
|
||||||
|
emitSpy = jest.spyOn(hc, 'emit');
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
emitSpy.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('down probes during the suppress window do not flip the badge', () => {
|
||||||
|
hc.recordStatus('svc1', makeUp());
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('up');
|
||||||
|
|
||||||
|
hc.suppressDuringDeploy('svc1');
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
hc.recordStatus('svc1', makeDown()); // well past DOWN_THRESHOLD=2
|
||||||
|
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('up');
|
||||||
|
const statusEmits = emitSpy.mock.calls.filter(c => c[0] === 'status-check');
|
||||||
|
expect(statusEmits.length).toBe(1); // only the bootstrap "up" emit
|
||||||
|
});
|
||||||
|
|
||||||
|
test('raw history keeps every probe during suppression', () => {
|
||||||
|
hc.recordStatus('svc1', makeUp());
|
||||||
|
hc.suppressDuringDeploy('svc1');
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
|
||||||
|
expect(hc.history.svc1.length).toBe(3);
|
||||||
|
expect(hc.currentStatus.get('svc1').status).toBe('down');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('no outage or slow-response incidents open while suppressed', () => {
|
||||||
|
hc.recordStatus('svc1', makeUp());
|
||||||
|
hc.suppressDuringDeploy('svc1');
|
||||||
|
const down = makeDown();
|
||||||
|
down.responseTime = 99999; // would trip slow-response too
|
||||||
|
hc.recordStatus('svc1', down);
|
||||||
|
|
||||||
|
expect(hc.incidents.length).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('after expiry, down probes flip the badge again (unchanged semantics)', () => {
|
||||||
|
hc.recordStatus('svc1', makeUp());
|
||||||
|
hc.suppressDuringDeploy('svc1', 1); // expires immediately
|
||||||
|
// spin clock past expiry without sleeps
|
||||||
|
hc.deploySuppressedUntil.set('svc1', Date.now() - 1);
|
||||||
|
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('down');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ttl is clamped to HEALTH_DEPLOY_SUPPRESS_MAX_MS', () => {
|
||||||
|
hc.suppressDuringDeploy('svc1', 10 * 60 * 60 * 1000); // 1h request
|
||||||
|
const until = hc.deploySuppressedUntil.get('svc1');
|
||||||
|
expect(until - Date.now()).toBeLessThanOrEqual(60000 + 50);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
/**
|
||||||
|
* DC-136 (judge round 2): ROUTE-level suppression ordering tests.
|
||||||
|
*
|
||||||
|
* The blocking issue on round 1: suppression was applied after awaiting
|
||||||
|
* the bridge operation — the noisy restart happens DURING that call, so
|
||||||
|
* the badge was never actually protected. These pins prove:
|
||||||
|
*
|
||||||
|
* 1. POST /deploy: suppressDuringDeploy fires BEFORE the bridge request
|
||||||
|
* is initiated (suppression is active while the bridge promise pends).
|
||||||
|
* 2. POST /rollback: same ordering.
|
||||||
|
* 3. On SUCCESS the window is cleared when the response returns.
|
||||||
|
* 4. On FAILURE the window is cleared too — a failed deploy must never
|
||||||
|
* start a fresh 10-minute silence (real downtime stays visible).
|
||||||
|
*
|
||||||
|
* supertest is lazy: the HTTP request only fires on .then()/end(). Each
|
||||||
|
* case attaches a no-op .then() immediately so the request is in flight
|
||||||
|
* while we assert on the pending-state ordering.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
|
||||||
|
const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc136-routes-'));
|
||||||
|
const TOKEN_FILE = path.join(TMP_DIR, 'bridge-token');
|
||||||
|
fs.writeFileSync(TOKEN_FILE, 'test-token-123');
|
||||||
|
|
||||||
|
process.env.SHIPDECK_BRIDGE_URL = 'http://127.0.0.1:8977';
|
||||||
|
process.env.SHIPDECK_BRIDGE_TOKEN_FILE = TOKEN_FILE;
|
||||||
|
|
||||||
|
// require AFTER env so the module-level consts pick the config up
|
||||||
|
const express = require('express');
|
||||||
|
const request = require('supertest');
|
||||||
|
const deploysRoutes = require('../routes/deploys');
|
||||||
|
const healthCheckerSingleton = require('../src/monitoring/health-checker');
|
||||||
|
|
||||||
|
function makeUp(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'up',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 200,
|
||||||
|
message: 'Service is healthy',
|
||||||
|
details: { headers: {}, bodyLength: 12 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDown(serviceId = 'svc1') {
|
||||||
|
return {
|
||||||
|
serviceId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
status: 'down',
|
||||||
|
responseTime: 50,
|
||||||
|
statusCode: 500,
|
||||||
|
message: 'fail',
|
||||||
|
details: { headers: {}, bodyLength: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||||
|
|
||||||
|
// Deterministic wait: poll until the calls log contains the given event kind
|
||||||
|
// (or timeout). Fixed sleeps race under parallel-jest load; this cannot.
|
||||||
|
async function waitForCall(calls, kind, timeoutMs = 2000) {
|
||||||
|
return waitForCallCount(calls, kind, 1, timeoutMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForCallCount(calls, kind, n, timeoutMs = 2000) {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
if (calls.filter(c => c[0] === kind).length >= n) return;
|
||||||
|
await sleep(5);
|
||||||
|
}
|
||||||
|
throw new Error(`timed out waiting for ${n}x '${kind}' in calls log`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the app with a controllable bridge. The bridge promise pends until
|
||||||
|
* `h.resolve()` is called; resolveWith lets a case choose the response.
|
||||||
|
*/
|
||||||
|
function makeHarness() {
|
||||||
|
const calls = []; // ordered event log: ['suppress', svc] | ['bridge', url] | ['clear', svc]
|
||||||
|
let resolveBridge;
|
||||||
|
let rejectBridge; // judge r4 polish: real promise-rejection path
|
||||||
|
const healthChecker = {
|
||||||
|
suppressDuringDeploy: (svc) => calls.push(['suppress', svc]),
|
||||||
|
clearDeploySuppression: (svc) => calls.push(['clear', svc]),
|
||||||
|
};
|
||||||
|
const fetchT = (url) => {
|
||||||
|
calls.push(['bridge', url]);
|
||||||
|
return new Promise((res, rej) => {
|
||||||
|
resolveBridge = res;
|
||||||
|
rejectBridge = rej;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const router = deploysRoutes({
|
||||||
|
asyncHandler: (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next),
|
||||||
|
log: { info: () => {}, warn: () => {}, error: () => {} },
|
||||||
|
auditLogger: undefined,
|
||||||
|
fetchT,
|
||||||
|
healthChecker,
|
||||||
|
});
|
||||||
|
const app = express();
|
||||||
|
app.use(express.json());
|
||||||
|
app.use('/api/v1/deploys', router);
|
||||||
|
return {
|
||||||
|
app,
|
||||||
|
calls,
|
||||||
|
resolve: () => resolveBridge({ status: 200, ok: true, json: async () => ({ ok: true, exit: 0, output: '' }) }),
|
||||||
|
resolveWith: (resp) => resolveBridge(resp),
|
||||||
|
// judge r4 polish: genuine promise rejection, not resolve-with-Error
|
||||||
|
reject: (err) => rejectBridge(err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DC-136 routes: suppression ordering vs the bridge call', () => {
|
||||||
|
test('deploy: suppressed BEFORE the bridge call, cleared on success', async () => {
|
||||||
|
const h = makeHarness();
|
||||||
|
const pending = request(h.app)
|
||||||
|
.post('/api/v1/deploys/deploy')
|
||||||
|
.send({ dir: '/root/demo-app', service: 'demo-app' });
|
||||||
|
pending.then(() => {}, () => {}); // fire the request NOW (supertest laziness)
|
||||||
|
await waitForCall(h.calls, 'bridge');
|
||||||
|
|
||||||
|
const suppressIdx = h.calls.findIndex(c => c[0] === 'suppress');
|
||||||
|
const bridgeIdx = h.calls.findIndex(c => c[0] === 'bridge');
|
||||||
|
expect(suppressIdx).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(bridgeIdx).toBeGreaterThan(suppressIdx); // ordering is the pin
|
||||||
|
expect(h.calls.find(c => c[0] === 'suppress')[1]).toBe('demo-app');
|
||||||
|
// still pending: no clear yet while the bridge promise hangs
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear')).toBe(false);
|
||||||
|
|
||||||
|
h.resolve();
|
||||||
|
const res = await pending;
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear' && c[1] === 'demo-app')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('deploy HTTP-failure: suppression is CLEARED, not renewed', async () => {
|
||||||
|
const h = makeHarness();
|
||||||
|
const pending = request(h.app)
|
||||||
|
.post('/api/v1/deploys/deploy')
|
||||||
|
.send({ dir: '/root/demo-app' });
|
||||||
|
pending.then(() => {}, () => {});
|
||||||
|
await waitForCall(h.calls, 'bridge');
|
||||||
|
expect(h.calls.some(c => c[0] === 'suppress')).toBe(true);
|
||||||
|
|
||||||
|
h.resolveWith({ status: 500, ok: false, json: async () => ({ ok: false, error: 'deploy failed' }) });
|
||||||
|
const res = await pending;
|
||||||
|
expect(res.status).toBe(502);
|
||||||
|
// failed deploy -> window cleared immediately; no fresh silence window
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rollback: suppressed BEFORE the bridge call, cleared on success', async () => {
|
||||||
|
const h = makeHarness();
|
||||||
|
const pending = request(h.app)
|
||||||
|
.post('/api/v1/deploys/rollback')
|
||||||
|
.send({ service: 'demo-app' });
|
||||||
|
pending.then(() => {}, () => {});
|
||||||
|
await waitForCall(h.calls, 'bridge');
|
||||||
|
|
||||||
|
const suppressIdx = h.calls.findIndex(c => c[0] === 'suppress');
|
||||||
|
const bridgeIdx = h.calls.findIndex(c => c[0] === 'bridge');
|
||||||
|
expect(suppressIdx).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(bridgeIdx).toBeGreaterThan(suppressIdx);
|
||||||
|
|
||||||
|
h.resolve();
|
||||||
|
const res = await pending;
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Judge r3 blockers: the bridge promise can REJECT (fetchT throw, network
|
||||||
|
// error, timeout). Cleanup must be guaranteed on that path too.
|
||||||
|
test('deploy with REJECTED bridge promise: suppression still cleared', async () => {
|
||||||
|
const h = makeHarness();
|
||||||
|
const pending = request(h.app)
|
||||||
|
.post('/api/v1/deploys/deploy')
|
||||||
|
.send({ dir: '/root/demo-app' });
|
||||||
|
pending.then(() => {}, () => {});
|
||||||
|
await waitForCall(h.calls, 'bridge');
|
||||||
|
expect(h.calls.some(c => c[0] === 'suppress')).toBe(true);
|
||||||
|
|
||||||
|
h.reject(new Error('ECONNREFUSED: bridge unreachable'));
|
||||||
|
const res = await pending;
|
||||||
|
expect(res.status).toBe(502);
|
||||||
|
// judge r4 polish: the 502 carries the original network error, proving
|
||||||
|
// this was a genuine fetch rejection (not a later parsing throw)
|
||||||
|
expect(JSON.stringify(res.body)).toContain('ECONNREFUSED');
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rollback with REJECTED bridge promise: suppression still cleared', async () => {
|
||||||
|
const h = makeHarness();
|
||||||
|
const pending = request(h.app)
|
||||||
|
.post('/api/v1/deploys/rollback')
|
||||||
|
.send({ service: 'demo-app' });
|
||||||
|
pending.then(() => {}, () => {});
|
||||||
|
await waitForCall(h.calls, 'bridge');
|
||||||
|
expect(h.calls.some(c => c[0] === 'suppress')).toBe(true);
|
||||||
|
|
||||||
|
h.reject(new Error('bridge timeout'));
|
||||||
|
const res = await pending;
|
||||||
|
expect(res.status).toBe(502);
|
||||||
|
expect(JSON.stringify(res.body)).toContain('bridge timeout');
|
||||||
|
expect(h.calls.some(c => c[0] === 'clear')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Judge r3 polish: overlapping deploys of the same service through ONE
|
||||||
|
// shared healthChecker — 2 suppresses, first clear must NOT end the
|
||||||
|
// window; only the second clear does (reference counting).
|
||||||
|
test('overlapping deploys: window survives until the last in-flight completes', async () => {
|
||||||
|
// One spy shared by both routers = the real singleton's role.
|
||||||
|
const shared = [];
|
||||||
|
const sharedHC = {
|
||||||
|
suppressDuringDeploy: (svc) => shared.push(['suppress', svc]),
|
||||||
|
clearDeploySuppression: (svc) => shared.push(['clear', svc]),
|
||||||
|
};
|
||||||
|
let resolveA;
|
||||||
|
let resolveB;
|
||||||
|
const mkRouter = (fetchT) => {
|
||||||
|
const r = deploysRoutes({
|
||||||
|
asyncHandler: (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next),
|
||||||
|
log: { info: () => {}, warn: () => {}, error: () => {} },
|
||||||
|
fetchT,
|
||||||
|
healthChecker: sharedHC,
|
||||||
|
});
|
||||||
|
const a = express();
|
||||||
|
a.use(express.json());
|
||||||
|
a.use('/api/v1/deploys', r);
|
||||||
|
return a;
|
||||||
|
};
|
||||||
|
const appA = mkRouter(() => new Promise(res => { resolveA = () => res({ status: 200, ok: true, json: async () => ({ ok: true, exit: 0, output: '' }) }); }));
|
||||||
|
const appB = mkRouter(() => new Promise(res => { resolveB = () => res({ status: 200, ok: true, json: async () => ({ ok: true, exit: 0, output: '' }) }); }));
|
||||||
|
|
||||||
|
const pa = request(appA).post('/api/v1/deploys/deploy').send({ dir: '/root/demo-app', service: 'demo-app' });
|
||||||
|
pa.then(() => {}, () => {});
|
||||||
|
await waitForCall(shared, 'suppress'); // first request is mid-flight
|
||||||
|
const pb = request(appB).post('/api/v1/deploys/deploy').send({ dir: '/root/demo-app' });
|
||||||
|
pb.then(() => {}, () => {});
|
||||||
|
await waitForCallCount(shared, 'suppress', 2); // second request too
|
||||||
|
|
||||||
|
expect(shared.filter(c => c[0] === 'suppress').length).toBe(2);
|
||||||
|
|
||||||
|
resolveA(); // first deploy completes -> clears its ref...
|
||||||
|
await pa;
|
||||||
|
expect(shared.filter(c => c[0] === 'clear').length).toBe(1);
|
||||||
|
|
||||||
|
resolveB(); // second (last) deploy completes -> clears the final ref
|
||||||
|
await pb;
|
||||||
|
expect(shared.filter(c => c[0] === 'clear').length).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
// The real singleton's ref-counting semantics (what the route spies above
|
||||||
|
// stub out): suppress->suppress->clear must leave the window ACTIVE.
|
||||||
|
test('real healthChecker: ref-counted suppress/clear keeps window until last clear', () => {
|
||||||
|
const hc = healthCheckerSingleton;
|
||||||
|
hc.displayedStatus = new Map();
|
||||||
|
hc.consecutiveSinceChange = new Map();
|
||||||
|
hc.currentStatus = new Map();
|
||||||
|
hc.history = {};
|
||||||
|
hc.deploySuppressedUntil = new Map();
|
||||||
|
hc.deploySuppressRefs = new Map();
|
||||||
|
hc.recordStatus('svc1', makeUp());
|
||||||
|
|
||||||
|
hc.suppressDuringDeploy('svc1');
|
||||||
|
hc.suppressDuringDeploy('svc1'); // overlapping second deploy
|
||||||
|
hc.clearDeploySuppression('svc1'); // first deploy finishes
|
||||||
|
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('up'); // still suppressed
|
||||||
|
|
||||||
|
hc.clearDeploySuppression('svc1'); // last deploy finishes
|
||||||
|
// window truly closed: post-hysteresis, two consecutive downs flip red
|
||||||
|
// (DOWN_THRESHOLD=2 — the suppressed probes correctly did NOT count
|
||||||
|
// toward the streak, and the displayed state resumes normal rules)
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('up'); // 1st down after clear
|
||||||
|
hc.recordStatus('svc1', makeDown());
|
||||||
|
expect(hc.displayedStatus.get('svc1').status).toBe('down'); // 2nd down flips
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
/**
|
||||||
|
* DC-134: data-driven login pages for registered-but-uncurated services.
|
||||||
|
*
|
||||||
|
* Before: /api/v1/auth/login-page served curated auto-login pages for
|
||||||
|
* {chat, plex, jellyfin, emby, sec} and 404'd for every other service —
|
||||||
|
* meaning every shipdeck/App-Selector install needed a code change
|
||||||
|
* (sso-gate.js edit + API restart) before its gated auto-login worked.
|
||||||
|
*
|
||||||
|
* After: any service registered in services.json gets a generic gated
|
||||||
|
* auto-login page (session pre-verified by the SHELL, then ?direct=1 to
|
||||||
|
* bypass the Caddy @needsAutoLogin loop). Curated pages always win.
|
||||||
|
*
|
||||||
|
* buildLoginPage() is exercised directly — it's the unit that decides
|
||||||
|
* page rendering, and the route handler is a thin wrapper around it.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
// Route-level harness: replicate the minimal deps the sso-gate factory needs.
|
||||||
|
const express = require('express');
|
||||||
|
const request = require('supertest');
|
||||||
|
|
||||||
|
function createApp({ services }) {
|
||||||
|
const factory = require('../routes/auth/sso-gate');
|
||||||
|
const router = factory({
|
||||||
|
authManager: {},
|
||||||
|
totpConfig: { enabled: true },
|
||||||
|
session: { isValid: () => true },
|
||||||
|
asyncHandler: (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next),
|
||||||
|
errorResponse: (res, code, msg, extra = {}) => res.status(code).json({ success: false, error: msg, ...extra }),
|
||||||
|
log: { info: () => {}, warn: () => {}, error: () => {} },
|
||||||
|
getAppSession: () => null,
|
||||||
|
appSessionCache: new Map(),
|
||||||
|
credentialManager: { retrieve: async () => null },
|
||||||
|
fetchT: async () => { throw new Error('not used'); },
|
||||||
|
getServiceById: async () => null,
|
||||||
|
licenseManager: {
|
||||||
|
hasFeature: () => false,
|
||||||
|
requirePremium: () => (req, res, next) => next(),
|
||||||
|
},
|
||||||
|
servicesStateManager: { read: async () => services },
|
||||||
|
siteConfig: { dashboardHost: 'status.sami' },
|
||||||
|
});
|
||||||
|
const app = express();
|
||||||
|
app.use('/api/v1', router);
|
||||||
|
return app;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DC-134: data-driven login pages', () => {
|
||||||
|
const registeredOnly = [
|
||||||
|
{ id: 'demo-hi3', name: 'Demo Hi3', url: 'https://hi3.sami' },
|
||||||
|
{ id: 'chat', name: 'Chat', url: 'https://chat.sami' }, // curated + registered
|
||||||
|
];
|
||||||
|
|
||||||
|
test('registered service WITHOUT a curated page gets a generic gated page', async () => {
|
||||||
|
const res = await request(createApp({ services: registeredOnly }))
|
||||||
|
.get('/api/v1/auth/login-page?service=demo-hi3');
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.headers['content-type']).toMatch(/html/);
|
||||||
|
expect(res.text).toContain('Signing in to Demo Hi3...');
|
||||||
|
expect(res.text).toContain("go('/?direct=1')");
|
||||||
|
// the SHELL must still enforce the session pre-check
|
||||||
|
expect(res.text).toContain('totp/check-session');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('curated page wins over the data-driven fallback (chat)', async () => {
|
||||||
|
const res = await request(createApp({ services: registeredOnly }))
|
||||||
|
.get('/api/v1/auth/login-page?service=chat');
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.text).toContain('Signing in...'); // curated title, not "Signing in to Chat..."
|
||||||
|
expect(res.text).not.toContain('Signing in to Chat...');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('service id with digits/hyphens survives the sanitizer', async () => {
|
||||||
|
const res = await request(createApp({ services: registeredOnly }))
|
||||||
|
.get('/api/v1/auth/login-page?service=demo-hi3');
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('unknown service still returns 404 Unknown service', async () => {
|
||||||
|
const res = await request(createApp({ services: registeredOnly }))
|
||||||
|
.get('/api/v1/auth/login-page?service=nonexistent');
|
||||||
|
expect(res.status).toBe(404);
|
||||||
|
expect(res.text).toContain('Unknown service');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('services read failure degrades to curated-only behavior (404, no crash)', async () => {
|
||||||
|
const res = await request(createApp({ services: null }))
|
||||||
|
.get('/api/v1/auth/login-page?service=demo-hi3');
|
||||||
|
expect(res.status).toBe(404);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('service display name is HTML-escaped in the title', async () => {
|
||||||
|
const services = [{ id: 'xss', name: '<script>alert(1)</script>', url: 'https://xss.sami' }];
|
||||||
|
const res = await request(createApp({ services }))
|
||||||
|
.get('/api/v1/auth/login-page?service=xss');
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.text).not.toContain('<script>alert(1)</script>');
|
||||||
|
expect(res.text).toContain('<script>');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
/**
|
||||||
|
* DC-135: shipdeck journal → Security Center pipeline.
|
||||||
|
*
|
||||||
|
* The shipdeck CLI appends one JSON row per lifecycle event to
|
||||||
|
* /var/lib/shipdeck/journal.jsonl. startShipdeckWorker() tails that file
|
||||||
|
* and appends a source_type='shipdeck' security event for each deploy /
|
||||||
|
* rollback row, with severity mapped from the verify[] block.
|
||||||
|
*
|
||||||
|
* Tests run the REAL worker against a temp journal file (hermetic sink,
|
||||||
|
* same pattern as caddy-worker-pipeline-dc113.test.js). Assertions pin:
|
||||||
|
* - VALID_SOURCE_TYPES admits 'shipdeck' (store accepts, unknown rejected)
|
||||||
|
* - deploy rows ingest as notice/success with service + epoch metadata
|
||||||
|
* - failed verify[] rows escalate to error severity
|
||||||
|
* - non-lifecycle rows (health checks etc.) do NOT ingest
|
||||||
|
* - unparseable lines are skipped without killing the worker
|
||||||
|
* - first-start replay cap: an oversized pre-existing backlog is skipped
|
||||||
|
* to the tail window (offset set to size - 1 MiB), not fully ingested
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
|
||||||
|
const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc135-shipdeck-'));
|
||||||
|
const JOURNAL = path.join(TMP_DIR, 'journal.jsonl');
|
||||||
|
const DATA_DIR = path.join(TMP_DIR, 'data');
|
||||||
|
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||||
|
|
||||||
|
process.env.SHIPDECK_JOURNAL_FILE = JOURNAL;
|
||||||
|
process.env.SECURITY_EVENT_LOG_FILE = path.join(TMP_DIR, 'security-events.jsonl');
|
||||||
|
|
||||||
|
// point platformPaths.dataDir at the hermetic dir BEFORE requiring the module
|
||||||
|
jest.doMock('../platform-paths', () => ({ dataDir: DATA_DIR }), { virtual: true });
|
||||||
|
|
||||||
|
const { VALID_SOURCE_TYPES } = require('../src/security/event-store');
|
||||||
|
const storeModule = require('../src/security/event-store');
|
||||||
|
const workers = require('../src/security/event-workers');
|
||||||
|
|
||||||
|
const silence = { info: () => {}, warn: () => {}, error: () => {} };
|
||||||
|
|
||||||
|
function row(overrides = {}) {
|
||||||
|
return JSON.stringify(Object.assign({
|
||||||
|
time: '2026-09-16T10:00:00Z',
|
||||||
|
service: 'demo-hi3',
|
||||||
|
host: 'dns2',
|
||||||
|
epoch: 1789548000,
|
||||||
|
pkg_sha256: '',
|
||||||
|
duration_s: 35.93,
|
||||||
|
action: 'deploy',
|
||||||
|
spec: { host: 'dns2', unit: 'demo-hi3.service', port: 8953, record: 'hi3.sami', verify_http: 'https://hi3.sami/' },
|
||||||
|
verify: [{ check: 'systemd-active', ok: true, detail: 'active' }, { check: 'http-tailnet', ok: true, detail: 'HTTP 200' }],
|
||||||
|
}, overrides));
|
||||||
|
}
|
||||||
|
|
||||||
|
function shipdeckEvents() {
|
||||||
|
return storeModule.getStore({ log: silence }).query({ source_type: 'shipdeck', limit: 100 }).events;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitTicks(n = 3) {
|
||||||
|
// createTail polls every 1s; give the worker a few ticks to consume
|
||||||
|
await new Promise(r => setTimeout(r, n * 1100));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DC-135: shipdeck source in the Security Center', () => {
|
||||||
|
let worker;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
worker = workers.startShipdeckWorker({ log: silence });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
worker.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("store admits source_type 'shipdeck'", () => {
|
||||||
|
expect(VALID_SOURCE_TYPES.has('shipdeck')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a successful deploy row ingests as notice/success with metadata', async () => {
|
||||||
|
fs.appendFileSync(JOURNAL, row() + '\n');
|
||||||
|
await waitTicks();
|
||||||
|
const evs = shipdeckEvents();
|
||||||
|
expect(evs.length).toBeGreaterThanOrEqual(1);
|
||||||
|
const ev = evs.find(e => e.target === 'demo-hi3' && e.action === 'shipdeck.deploy');
|
||||||
|
expect(ev).toBeDefined();
|
||||||
|
expect(ev.severity).toBe('notice');
|
||||||
|
expect(ev.outcome).toBe('success');
|
||||||
|
expect(ev.source_host).toBe('dns2');
|
||||||
|
expect(ev.metadata.epoch).toBe(1789548000);
|
||||||
|
expect(ev.metadata.record).toBe('hi3.sami');
|
||||||
|
expect(Array.isArray(ev.metadata.verify)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a failed verify[] row escalates to error severity', async () => {
|
||||||
|
fs.appendFileSync(JOURNAL, row({
|
||||||
|
service: 'broken-app',
|
||||||
|
action: 'rollback',
|
||||||
|
verify: [{ check: 'systemd-active', ok: false, detail: 'failed' }],
|
||||||
|
}) + '\n');
|
||||||
|
await waitTicks();
|
||||||
|
const ev = shipdeckEvents().find(e => e.target === 'broken-app' && e.action === 'shipdeck.rollback');
|
||||||
|
expect(ev).toBeDefined();
|
||||||
|
expect(ev.severity).toBe('error');
|
||||||
|
expect(ev.outcome).toBe('error');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('non-lifecycle rows (health checks) do not ingest', async () => {
|
||||||
|
const before = shipdeckEvents().length;
|
||||||
|
fs.appendFileSync(JOURNAL, row({ service: 'demo-hi3', action: 'health', verify: [] }) + '\n');
|
||||||
|
fs.appendFileSync(JOURNAL, 'not-json-at-all\n');
|
||||||
|
await waitTicks();
|
||||||
|
expect(shipdeckEvents().length).toBe(before);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('first-start replay cap: oversized backlog is skipped to the tail window', async () => {
|
||||||
|
// Judge r3: hermetic restart — fresh journal path (env read at worker
|
||||||
|
// start), fresh offset file (so this is a genuine first start), and
|
||||||
|
// delta-based assertions on the shared store singleton.
|
||||||
|
worker.stop();
|
||||||
|
|
||||||
|
const BIG = path.join(TMP_DIR, 'journal-big.jsonl');
|
||||||
|
const offsetFile = path.join(DATA_DIR, '.shipdeck-tail-offset');
|
||||||
|
if (fs.existsSync(offsetFile)) fs.rmSync(offsetFile);
|
||||||
|
|
||||||
|
// Build a backlog > firstStartMaxBytes (1 MiB) of lifecycle rows that
|
||||||
|
// WOULD all ingest without the cap; the final row carries a distinct
|
||||||
|
// service name so we can prove the tail window itself was processed.
|
||||||
|
const pad = row({ service: 'oldsvc', epoch: 1 }) + '\n';
|
||||||
|
const need = Math.ceil((2 * 1024 * 1024) / pad.length);
|
||||||
|
let out = '';
|
||||||
|
for (let i = 0; i < need; i++) out += pad;
|
||||||
|
out += row({ service: 'tailsvc', epoch: 2 }) + '\n';
|
||||||
|
fs.writeFileSync(BIG, out);
|
||||||
|
|
||||||
|
const prevJournal = process.env.SHIPDECK_JOURNAL_FILE;
|
||||||
|
process.env.SHIPDECK_JOURNAL_FILE = BIG;
|
||||||
|
const deltaBefore = shipdeckEvents().length;
|
||||||
|
const w2 = workers.startShipdeckWorker({ log: silence });
|
||||||
|
try {
|
||||||
|
await waitTicks(4);
|
||||||
|
} finally {
|
||||||
|
w2.stop();
|
||||||
|
process.env.SHIPDECK_JOURNAL_FILE = prevJournal;
|
||||||
|
fs.rmSync(BIG, { force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const delta = shipdeckEvents().length - deltaBefore;
|
||||||
|
// cap proof: a 2MB backlog must not become `need` events (that would
|
||||||
|
// mean the whole pre-existing file was replayed on first start)
|
||||||
|
expect(delta).toBeLessThan(need);
|
||||||
|
expect(delta).toBeGreaterThan(0); // tail window still ingested
|
||||||
|
// and specifically the tail of the file made it in
|
||||||
|
expect(shipdeckEvents().some(e => e.target === 'tailsvc')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -267,14 +267,22 @@ module.exports = function(deps) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Serve service-specific auto-login page (auth enforced by Caddy forward_auth upstream)
|
// Serve service-specific auto-login page (auth enforced by Caddy forward_auth upstream)
|
||||||
router.get('/auth/login-page', (req, res) => {
|
router.get('/auth/login-page', asyncHandler(async (req, res) => {
|
||||||
const service = (req.query.service || '').replace(/[^a-z]/g, '');
|
// DC-134: ids may contain digits and hyphens (shipdeck installs like
|
||||||
|
// demo-hi3) — keep them, strip everything else. The value is only ever
|
||||||
|
// compared against the curated page keys and service ids.
|
||||||
|
const service = (req.query.service || '').replace(/[^a-z0-9-]/g, '');
|
||||||
const configuredHost = siteConfig?.dashboardHost;
|
const configuredHost = siteConfig?.dashboardHost;
|
||||||
const dashboardOrigin = typeof configuredHost === 'string'
|
const dashboardOrigin = typeof configuredHost === 'string'
|
||||||
&& /^[a-zA-Z0-9][a-zA-Z0-9.-]*$/.test(configuredHost)
|
&& /^[a-zA-Z0-9][a-zA-Z0-9.-]*$/.test(configuredHost)
|
||||||
? `https://${configuredHost}`
|
? `https://${configuredHost}`
|
||||||
: 'https://status.sami';
|
: 'https://status.sami';
|
||||||
const html = buildLoginPage(service, dashboardOrigin);
|
// DC-134: read the live services list so any registered service without a
|
||||||
|
// curated auto-login flow still gets a gated generic login page instead
|
||||||
|
// of a 404. Read failure falls back to curated-only behavior.
|
||||||
|
let services = null;
|
||||||
|
try { services = await servicesStateManager.read(); } catch (_) { services = null; }
|
||||||
|
const html = buildLoginPage(service, dashboardOrigin, services);
|
||||||
if (!html) return res.status(404).send('Unknown service');
|
if (!html) return res.status(404).send('Unknown service');
|
||||||
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
||||||
res.setHeader('Cache-Control', 'no-store');
|
res.setHeader('Cache-Control', 'no-store');
|
||||||
@@ -287,12 +295,12 @@ module.exports = function(deps) {
|
|||||||
// one response only; every other route keeps the strict app-wide policy.
|
// one response only; every other route keeps the strict app-wide policy.
|
||||||
res.setHeader('Content-Security-Policy', "default-src 'self'; style-src 'self'; script-src 'self' 'unsafe-inline'; img-src 'self' data: https:; connect-src 'self'; font-src 'self' data:; object-src 'none'; media-src 'self'; frame-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'self'");
|
res.setHeader('Content-Security-Policy', "default-src 'self'; style-src 'self'; script-src 'self' 'unsafe-inline'; img-src 'self' data: https:; connect-src 'self'; font-src 'self' data:; object-src 'none'; media-src 'self'; frame-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'self'");
|
||||||
res.send(html);
|
res.send(html);
|
||||||
});
|
}));
|
||||||
|
|
||||||
return router;
|
return router;
|
||||||
};
|
};
|
||||||
|
|
||||||
function buildLoginPage(service, dashboardOrigin = 'https://status.sami') {
|
function buildLoginPage(service, dashboardOrigin = 'https://status.sami', services = null) {
|
||||||
// Pre-auth check via <meta http-equiv="refresh"> so it fires even when JS is
|
// Pre-auth check via <meta http-equiv="refresh"> so it fires even when JS is
|
||||||
// disabled or blocked. The cookie is sent automatically because we hit the
|
// disabled or blocked. The cookie is sent automatically because we hit the
|
||||||
// same origin (plex.sami); if the API returns 200 the user has a valid
|
// same origin (plex.sami); if the API returns 200 the user has a valid
|
||||||
@@ -401,7 +409,38 @@ ft('chat').then(function(r){return r.text()}).then(function(t){
|
|||||||
};
|
};
|
||||||
|
|
||||||
const cfg = pages[service];
|
const cfg = pages[service];
|
||||||
if (!cfg) return null;
|
if (!cfg) {
|
||||||
|
// DC-134: data-driven fallback. Any service registered in services.json
|
||||||
|
// (App Selector install, DC-131 git install, UI add) gets a generic gated
|
||||||
|
// auto-login page — session was already verified by the SHELL above, so
|
||||||
|
// the body just enters the app the same way the `sec` page does.
|
||||||
|
// ?direct=1 bypasses the Caddy @needsAutoLogin redirect loop. Curated
|
||||||
|
// pages above always win; unknown services still 404 below.
|
||||||
|
const registered = Array.isArray(services) &&
|
||||||
|
services.some(s => s && (s.id === service || s.subdomain === service));
|
||||||
|
if (registered) {
|
||||||
|
const name = (() => {
|
||||||
|
const s = services.find(x => x && (x.id === service || x.subdomain === service));
|
||||||
|
const raw = (s && typeof s.name === 'string' && s.name) || service;
|
||||||
|
// HTML-safe: the title is interpolated into the page shell.
|
||||||
|
return String(raw).replace(/[&<>"']/g, c => (
|
||||||
|
{ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]
|
||||||
|
));
|
||||||
|
})();
|
||||||
|
const fallback = {
|
||||||
|
title: `Signing in to ${name}...`,
|
||||||
|
bg: '#0a0a0a',
|
||||||
|
accent: '#60a5fa',
|
||||||
|
body: `d.textContent='Session verified, opening dashboard...';go('/?direct=1');`,
|
||||||
|
};
|
||||||
|
return SHELL(fallback.body)
|
||||||
|
.replace(/__TITLE__/g, fallback.title)
|
||||||
|
.replace('__BG__', fallback.bg)
|
||||||
|
.replace('__ACCENT__', fallback.accent)
|
||||||
|
.replace('__DASHBOARD_ORIGIN__', JSON.stringify(dashboardOrigin));
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
return SHELL(cfg.body)
|
return SHELL(cfg.body)
|
||||||
.replace(/__TITLE__/g, cfg.title)
|
.replace(/__TITLE__/g, cfg.title)
|
||||||
.replace('__BG__', cfg.bg)
|
.replace('__BG__', cfg.bg)
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ function readBridgeToken() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = function ({ asyncHandler, log, auditLogger, fetchT }) {
|
module.exports = function ({ asyncHandler, log, auditLogger, fetchT, healthChecker }) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
function featureEnabled() {
|
function featureEnabled() {
|
||||||
@@ -151,8 +151,29 @@ module.exports = function ({ asyncHandler, log, auditLogger, fetchT }) {
|
|||||||
if (typeof dir !== 'string' || !dir.trim()) {
|
if (typeof dir !== 'string' || !dir.trim()) {
|
||||||
return errorResponse(res, 400, 'dir is required');
|
return errorResponse(res, 400, 'dir is required');
|
||||||
}
|
}
|
||||||
|
const serviceNames = (() => {
|
||||||
|
// The deploy dir may host a differently-named service; suppress the
|
||||||
|
// dir basename plus whatever service name the panel passed alongside
|
||||||
|
// the request (deploy payload convention), covering naming skew.
|
||||||
|
const base = String(dir).replace(/\/+$/, '').split('/').pop() || '';
|
||||||
|
const requested = typeof (req.body && req.body.service) === 'string' ? req.body.service : '';
|
||||||
|
return [...new Set([base, requested])].filter(n => n && SERVICE_RE.test(n));
|
||||||
|
})();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { status, body } = await bridge('POST', '/api/deploy', { dir }, SHIPDECK_DEPLOY_TIMEOUT);
|
// DC-136: suppress BEFORE initiating — the restart blackholes probes
|
||||||
|
// DURING the bridge call, not after it resolves.
|
||||||
|
if (healthChecker) serviceNames.forEach(n => healthChecker.suppressDuringDeploy(n));
|
||||||
|
let bridgeResult;
|
||||||
|
try {
|
||||||
|
bridgeResult = await bridge('POST', '/api/deploy', { dir }, SHIPDECK_DEPLOY_TIMEOUT);
|
||||||
|
} finally {
|
||||||
|
// Judge r3: guaranteed cleanup on EVERY exit path — success, HTTP
|
||||||
|
// failure, thrown fetch error. A failed deploy never leaves real
|
||||||
|
// downtime hidden behind a suppression window.
|
||||||
|
if (healthChecker) serviceNames.forEach(n => healthChecker.clearDeploySuppression(n));
|
||||||
|
}
|
||||||
|
const { status, body } = bridgeResult;
|
||||||
if (auditLogger) {
|
if (auditLogger) {
|
||||||
auditLogger.log({
|
auditLogger.log({
|
||||||
action: 'deploy.shipdeck',
|
action: 'deploy.shipdeck',
|
||||||
@@ -179,7 +200,17 @@ module.exports = function ({ asyncHandler, log, auditLogger, fetchT }) {
|
|||||||
return errorResponse(res, 400, 'invalid service name');
|
return errorResponse(res, 400, 'invalid service name');
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const { status, body } = await bridge('POST', '/api/rollback', { service }, SHIPDECK_DEPLOY_TIMEOUT);
|
// DC-136: suppress BEFORE the rollback restarts the unit (same shape
|
||||||
|
// as /deploy); clear in a finally so failed rollbacks and thrown
|
||||||
|
// bridge errors never hide real downtime.
|
||||||
|
if (healthChecker) healthChecker.suppressDuringDeploy(service);
|
||||||
|
let bridgeResult;
|
||||||
|
try {
|
||||||
|
bridgeResult = await bridge('POST', '/api/rollback', { service }, SHIPDECK_DEPLOY_TIMEOUT);
|
||||||
|
} finally {
|
||||||
|
if (healthChecker) healthChecker.clearDeploySuppression(service);
|
||||||
|
}
|
||||||
|
const { status, body } = bridgeResult;
|
||||||
if (auditLogger) {
|
if (auditLogger) {
|
||||||
auditLogger.log({
|
auditLogger.log({
|
||||||
action: 'deploy.rollback',
|
action: 'deploy.rollback',
|
||||||
|
|||||||
@@ -786,6 +786,7 @@ async function createApp() {
|
|||||||
log: ctx.log,
|
log: ctx.log,
|
||||||
auditLogger: ctx.auditLogger,
|
auditLogger: ctx.auditLogger,
|
||||||
fetchT: ctx.fetchT,
|
fetchT: ctx.fetchT,
|
||||||
|
healthChecker, // DC-136: deploy-aware badge suppression
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Log Insights — plain English activity summary + safe log disposal
|
// Log Insights — plain English activity summary + safe log disposal
|
||||||
|
|||||||
@@ -56,6 +56,10 @@ function readPositiveIntEnv(name, fallback) {
|
|||||||
|
|
||||||
const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2);
|
const DOWN_THRESHOLD = readPositiveIntEnv('HEALTH_DOWN_THRESHOLD', 2);
|
||||||
const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1);
|
const UP_THRESHOLD = readPositiveIntEnv('HEALTH_UP_THRESHOLD', 1);
|
||||||
|
// DC-136: max time a deploy-suppression flag may hold a badge. A shipdeck
|
||||||
|
// deploy blackholes probes for seconds to a couple of minutes; the flag
|
||||||
|
// auto-expires so a crashed deploy can never silence a badge forever.
|
||||||
|
const DEPLOY_SUPPRESS_MAX_MS = readPositiveIntEnv('HEALTH_DEPLOY_SUPPRESS_MAX_MS', 10 * 60 * 1000);
|
||||||
|
|
||||||
class HealthChecker extends EventEmitter {
|
class HealthChecker extends EventEmitter {
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -84,6 +88,59 @@ class HealthChecker extends EventEmitter {
|
|||||||
// HIGHER generation than the captured one marks the capture as stale. Entry
|
// HIGHER generation than the captured one marks the capture as stale. Entry
|
||||||
// is deleted when the service is removed, so the live map cannot leak.
|
// is deleted when the service is removed, so the live map cannot leak.
|
||||||
this.removedGenerations = new Map();
|
this.removedGenerations = new Map();
|
||||||
|
// DC-136: serviceId -> expires-at (ms epoch). While active and unexpired,
|
||||||
|
// probes that land during a shipdeck deploy/rollback are recorded in raw
|
||||||
|
// history but don't drive the displayed badge or incident transitions.
|
||||||
|
this.deploySuppressedUntil = new Map();
|
||||||
|
// DC-136 r3 (judge polish): active-suppression reference counts so
|
||||||
|
// overlapping deploy requests for the same service can't clear each
|
||||||
|
// other's window while one of them is still in flight.
|
||||||
|
this.deploySuppressRefs = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DC-136: mark a service as mid-deploy. While suppressed, probe results
|
||||||
|
* still land in history (full fidelity preserved) but do NOT flip the
|
||||||
|
* displayed badge or open/resolve outage incidents — a service that is
|
||||||
|
* momentarily blackholed by its own redeploy should not page anyone.
|
||||||
|
* Suppression auto-expires after HEALTH_DEPLOY_SUPPRESS_MAX_MS (10 min)
|
||||||
|
* so a crashed deploy cannot silence a badge forever.
|
||||||
|
*/
|
||||||
|
suppressDuringDeploy(serviceId, ttlMs) {
|
||||||
|
const ttl = Number.isSafeInteger(ttlMs) && ttlMs > 0 ? ttlMs : DEPLOY_SUPPRESS_MAX_MS;
|
||||||
|
this.deploySuppressedUntil.set(serviceId, Date.now() + Math.min(ttl, DEPLOY_SUPPRESS_MAX_MS));
|
||||||
|
this.deploySuppressRefs.set(serviceId, (this.deploySuppressRefs.get(serviceId) || 0) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DC-136 (judge round 2): end the suppression window explicitly. The
|
||||||
|
* deploys routes call this when the bridge operation COMPLETES — on
|
||||||
|
* success because shipdeck health-verified the service before returning,
|
||||||
|
* and on failure because real downtime must become visible immediately
|
||||||
|
* (a failed deploy must never start a fresh 10-minute silence window).
|
||||||
|
* Reference-counted (judge round 3): with overlapping deploys of the
|
||||||
|
* same service, the window survives until the LAST in-flight request
|
||||||
|
* finishes. Always invoked from a `finally` so a thrown bridge error
|
||||||
|
* can never leak an active suppression window.
|
||||||
|
*/
|
||||||
|
clearDeploySuppression(serviceId) {
|
||||||
|
const refs = (this.deploySuppressRefs.get(serviceId) || 0) - 1;
|
||||||
|
if (refs > 0) {
|
||||||
|
this.deploySuppressRefs.set(serviceId, refs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.deploySuppressRefs.delete(serviceId);
|
||||||
|
this.deploySuppressedUntil.delete(serviceId);
|
||||||
|
}
|
||||||
|
|
||||||
|
_isDeploySuppressed(serviceId) {
|
||||||
|
const until = this.deploySuppressedUntil.get(serviceId);
|
||||||
|
if (until === undefined) return false;
|
||||||
|
if (Date.now() >= until) {
|
||||||
|
this.deploySuppressedUntil.delete(serviceId);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -432,6 +489,15 @@ class HealthChecker extends EventEmitter {
|
|||||||
// _computeDisplayedStatus compares the raw probe against the DISPLAYED
|
// _computeDisplayedStatus compares the raw probe against the DISPLAYED
|
||||||
// status (not the previous raw status), so the "consecutive since
|
// status (not the previous raw status), so the "consecutive since
|
||||||
// change" counter doesn't depend on the order of writes here.
|
// change" counter doesn't depend on the order of writes here.
|
||||||
|
// DC-136: during a shipdeck deploy/rollback window the probe result is
|
||||||
|
// still recorded (history + currentStatus above stay full-fidelity) but
|
||||||
|
// must not drive the badge — a redeploy blackholes the service for
|
||||||
|
// seconds and the red flip would be pure deploy noise. The displayed
|
||||||
|
// map is left untouched; the badge simply holds its pre-deploy state.
|
||||||
|
if (this._isDeploySuppressed(serviceId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const displayed = this._computeDisplayedStatus(serviceId, status);
|
const displayed = this._computeDisplayedStatus(serviceId, status);
|
||||||
const previousDisplayed = this.displayedStatus.get(serviceId);
|
const previousDisplayed = this.displayedStatus.get(serviceId);
|
||||||
const displayChanged =
|
const displayChanged =
|
||||||
@@ -456,7 +522,13 @@ class HealthChecker extends EventEmitter {
|
|||||||
* Check for incidents (downtime, slow response, etc.)
|
* Check for incidents (downtime, slow response, etc.)
|
||||||
*/
|
*/
|
||||||
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
checkForIncidents(serviceId, status, config, previous = this.currentStatus.get(serviceId), previousDisplayed = null) {
|
||||||
|
// DC-136: probe results inside a deploy-suppression window are deploy
|
||||||
|
// noise by definition (timeouts, 5xx from a restarting unit, huge
|
||||||
|
// response times) — they must not open outage/slow-response incidents.
|
||||||
|
// Slow-response and SLA checks are included in the skip; SLA math runs
|
||||||
|
// on history uptime which is unaffected by this early return.
|
||||||
|
if (this._isDeploySuppressed(serviceId)) return;
|
||||||
|
|
||||||
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
// DC-090: outage incidents follow the DISPLAYED (post-hysteresis) status —
|
||||||
// the same signal that flips the dashboard badge. A single raw "down"
|
// the same signal that flips the dashboard badge. A single raw "down"
|
||||||
// blip that hysteresis suppresses must not open a critical outage
|
// blip that hysteresis suppresses must not open a critical outage
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ const TRIM_TARGET_FACTOR = 0.8; // post-trim target: ≤80% of the byte budget
|
|||||||
const DEFAULT_TRIM_SIZE_LIMIT = parseInt(
|
const DEFAULT_TRIM_SIZE_LIMIT = parseInt(
|
||||||
process.env.SECURITY_EVENT_TRIM_BYTES || String(50 * 1024 * 1024), 10);
|
process.env.SECURITY_EVENT_TRIM_BYTES || String(50 * 1024 * 1024), 10);
|
||||||
|
|
||||||
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent']);
|
const VALID_SOURCE_TYPES = new Set(['api', 'caddy', 'fail2ban', 'shared-bans', 'syslog', 'agent', 'shipdeck']);
|
||||||
const VALID_SEVERITIES = new Set(['info', 'notice', 'warn', 'error', 'critical']);
|
const VALID_SEVERITIES = new Set(['info', 'notice', 'warn', 'error', 'critical']);
|
||||||
const VALID_OUTCOMES = new Set(['success', 'failure', 'denied', 'blocked', 'rate-limited', 'error', 'unknown']);
|
const VALID_OUTCOMES = new Set(['success', 'failure', 'denied', 'blocked', 'rate-limited', 'error', 'unknown']);
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,11 @@
|
|||||||
* 3. fail2ban log tail — parses /var/log/fail2ban.log for ban/unban
|
* 3. fail2ban log tail — parses /var/log/fail2ban.log for ban/unban
|
||||||
* actions. SSH jail is the default; can extend to other jails.
|
* actions. SSH jail is the default; can extend to other jails.
|
||||||
*
|
*
|
||||||
|
* 4. shipdeck journal tail — parses /var/lib/shipdeck/journal.jsonl
|
||||||
|
* (JSONL, one row per lifecycle event, DC-135). Deploy/rollback rows
|
||||||
|
* become source_type='shipdeck' security events — an unexplained
|
||||||
|
* redeploy is a security-relevant event.
|
||||||
|
*
|
||||||
* Each worker:
|
* Each worker:
|
||||||
* - Starts on app boot (via server.js)
|
* - Starts on app boot (via server.js)
|
||||||
* - Tracks its byte offset in the log file so it survives restarts (no re-emit)
|
* - Tracks its byte offset in the log file so it survives restarts (no re-emit)
|
||||||
@@ -422,6 +427,82 @@ function startFail2banWorker({ log } = {}) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DC-135: shipdeck journal tail worker. The shipdeck CLI appends one JSON
|
||||||
|
* row per deploy/rollback lifecycle event to /var/lib/shipdeck/journal.jsonl
|
||||||
|
* (format documented in the shipdeck repo: time, service, host, epoch,
|
||||||
|
* action, duration_s, spec, verify[]). Tail it so deploys appear in the
|
||||||
|
* Security Center timeline next to auth and perimeter events — an
|
||||||
|
* unexplained redeploy IS a security-relevant event. We ingest only
|
||||||
|
* lifecycle actions (deploy/rollback), not health probes, so the store
|
||||||
|
* isn't flooded by routine checks.
|
||||||
|
*
|
||||||
|
* Severity mapping:
|
||||||
|
* deploy/rollback success -> notice (infrastructure changed)
|
||||||
|
* deploy/rollback failure -> error
|
||||||
|
* unknown/unexpected action -> notice
|
||||||
|
*/
|
||||||
|
function startShipdeckWorker({ log: logger = log } = {}) {
|
||||||
|
const journalPath = process.env.SHIPDECK_JOURNAL_FILE || '/var/lib/shipdeck/journal.jsonl';
|
||||||
|
const stateFile = path.join(platformPaths.dataDir, '.shipdeck-tail-offset');
|
||||||
|
const store = getStore({ log: logger });
|
||||||
|
|
||||||
|
const LIFECYCLE_ACTIONS = new Set(['deploy', 'rollback']);
|
||||||
|
|
||||||
|
let missingWarned = false;
|
||||||
|
function warnIfMissing() {
|
||||||
|
if (missingWarned) return;
|
||||||
|
fs.stat(journalPath, (err) => {
|
||||||
|
if (!err) return;
|
||||||
|
missingWarned = true;
|
||||||
|
logger.warn?.('events', `shipdeck journal not found at ${journalPath} — shipdeck-source security events disabled (set SHIPDECK_JOURNAL_FILE)`, { worker: 'shipdeck' });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
warnIfMissing();
|
||||||
|
|
||||||
|
return createTail({
|
||||||
|
filePath: journalPath,
|
||||||
|
stateFile,
|
||||||
|
label: 'shipdeck',
|
||||||
|
firstStartMaxBytes: 1 * 1024 * 1024,
|
||||||
|
onAppear: () => {
|
||||||
|
logger.info?.('events', `shipdeck journal active at ${journalPath} — shipdeck-source security events enabled`, { worker: 'shipdeck' });
|
||||||
|
},
|
||||||
|
onLine: (line) => {
|
||||||
|
let row;
|
||||||
|
try { row = JSON.parse(line); }
|
||||||
|
catch { return; } // journal is JSONL; skip torn/unparseable lines
|
||||||
|
if (!row || typeof row !== 'object') return;
|
||||||
|
const action = typeof row.action === 'string' ? row.action : 'unknown';
|
||||||
|
// Health/verify/lifecycle-noise rows are not security events.
|
||||||
|
if (!LIFECYCLE_ACTIONS.has(action)) return;
|
||||||
|
const okAll = Array.isArray(row.verify) && row.verify.length > 0
|
||||||
|
? row.verify.every(v => v && v.ok === true)
|
||||||
|
: true; // no verify block -> treat as accepted (local_mode rows)
|
||||||
|
const failed = okAll === false || (typeof row.error === 'string' && row.error.length > 0);
|
||||||
|
store.append({
|
||||||
|
source_host: typeof row.host === 'string' && row.host ? row.host : HOSTNAME,
|
||||||
|
source_type: 'shipdeck',
|
||||||
|
actor: null,
|
||||||
|
target: typeof row.service === 'string' ? row.service : null,
|
||||||
|
action: `shipdeck.${action}`,
|
||||||
|
outcome: failed ? 'error' : 'success',
|
||||||
|
severity: failed ? 'error' : 'notice',
|
||||||
|
message: failed
|
||||||
|
? `shipdeck ${action} of ${row.service} FAILED (epoch ${row.epoch})`
|
||||||
|
: `shipdeck ${action} of ${row.service} succeeded (epoch ${row.epoch}, ${(row.duration_s || 0).toFixed ? row.duration_s.toFixed(1) : row.duration_s}s)`,
|
||||||
|
metadata: {
|
||||||
|
epoch: row.epoch || null,
|
||||||
|
duration_s: row.duration_s || null,
|
||||||
|
record: (row.spec && row.spec.record) || null,
|
||||||
|
verify_http: (row.spec && row.spec.verify_http) || null,
|
||||||
|
verify: Array.isArray(row.verify) ? row.verify : null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start all workers. Returns a stop function that shuts them all down.
|
* Start all workers. Returns a stop function that shuts them all down.
|
||||||
*/
|
*/
|
||||||
@@ -433,6 +514,8 @@ function startAll({ log } = {}) {
|
|||||||
catch (e) { log.error('events', e, { worker: 'shared_bans', phase: 'start' }); }
|
catch (e) { log.error('events', e, { worker: 'shared_bans', phase: 'start' }); }
|
||||||
try { workers.push(startFail2banWorker({ log })); }
|
try { workers.push(startFail2banWorker({ log })); }
|
||||||
catch (e) { log.error('events', e, { worker: 'fail2ban', phase: 'start' }); }
|
catch (e) { log.error('events', e, { worker: 'fail2ban', phase: 'start' }); }
|
||||||
|
try { workers.push(startShipdeckWorker({ log })); }
|
||||||
|
catch (e) { log.error('events', e, { worker: 'shipdeck', phase: 'start' }); }
|
||||||
return {
|
return {
|
||||||
stop() { workers.forEach(w => { try { w.stop(); } catch {} }); },
|
stop() { workers.forEach(w => { try { w.stop(); } catch {} }); },
|
||||||
workers,
|
workers,
|
||||||
@@ -444,6 +527,7 @@ module.exports = {
|
|||||||
startCaddyWorker,
|
startCaddyWorker,
|
||||||
startSharedBansWorker,
|
startSharedBansWorker,
|
||||||
startFail2banWorker,
|
startFail2banWorker,
|
||||||
|
startShipdeckWorker,
|
||||||
startAll,
|
startAll,
|
||||||
resolveCaddyAction,
|
resolveCaddyAction,
|
||||||
};
|
};
|
||||||
Reference in New Issue
Block a user