fix(security): event-store retention + race + total-cap defects (DC-116) [glm-grade=B]

- query().total: full-scan true count (was capped at offset+limit by an early break — dashboard 24h stat read ≤1000 vs real ~46k) - trim trigger/curer mismatch: byte trigger + line curer never converged when avg line > ~524B (live avg 355B); now keeps maxDisk lines OR ≤80% byte budget, whichever retains fewer (TRIM_TARGET_FACTOR); budget env-overridable via SECURITY_EVENT_TRIM_BYTES / opts.trimSizeLimit - trim/append race: trim renamed over the file mid-append losing events to the unlinked inode; now single-flight, queue-empty gated, holds the write lock, error paths always release + lazily re-kick (no hot loop) Judge: glm-4.6@zai-coding-paas adversarial cold-read, grade B clean (0 blocking, 4 polish — 2 folded, 1 already-satisfied, 1 deferred as judge-endorsed safer). 127 suites / 2849 tests green.
2026-08-23 15:02:00 -07:00
parent 65457ff8e0
commit 2dce6dca5e
2 changed files with 325 additions and 28 deletions
@@ -0,0 +1,231 @@
+/**
+ * DC-116 regression pins — security event store retention + query.total.
+ *
+ * Background (2026-08-23, one day after DC-113 activated the caddy source):
+ * live store had 46,494 events (16.5MB) growing ~2MB/day. Cold review of
+ * src/security/event-store.js found three defects:
+ *
+ *   1. query().total lied: the scan broke at offset+limit, so `total` was
+ *      capped at the page size (<=1000). LIVE user-facing impact — the
+ *      dashboard "N events (24h)" stat (status/js/security-center.js reads
+ *      data.total) and GET /hosts/:id/health events_24h showed 1000 when
+ *      the real 24h count was tens of thousands.
+ *   2. Trim trigger/curer mismatch: trigger was byte-based (>50MB) but the
+ *      curer was line-count-based (no-op unless >maxDisk=100k lines). If the
+ *      average line ever exceeded ~524B (50MB/100k — 0.5% of live lines were
+ *      already >524B, scanner bursts inflate metadata), trim fired on every
+ *      append and rewrote nothing — unbounded file + full-file re-read on
+ *      the write path.
+ *   3. Trim/append race: trim renamed over the file with appends in flight;
+ *      events appended after trim's readFile landed on the unlinked inode
+ *      and were silently lost.
+ *
+ * Tests use the REAL store with temp files. No mocks of the module under test.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+
+// Hermetic sinks (same pattern as caddy-worker-pipeline-dc113.test.js)
+const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc116-store-'));
+process.env.SECURITY_EVENT_LOG_FILE = path.join(TMP_DIR, 'security-events.jsonl');
+
+const { SecurityEventStore } = require('../src/security/event-store');
+
+const silence = { info: () => {}, warn: () => {}, error: () => {} };
+
+function makeStore(opts = {}) {
+  return new SecurityEventStore({
+    log: silence,
+    filePath: path.join(TMP_DIR, `store-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`),
+    ...opts,
+  });
+}
+
+// Deterministic event factory. `target` carries the unique marker — it is
+// never overridden by the fat-payload tests, which replace `message`.
+function ev(n, over = {}) {
+  return {
+    source_type: 'api',
+    actor: `actor-${n % 5}`,
+    action: `action-${n % 3}`,
+    target: `t-${n}`,
+    outcome: 'success',
+    severity: 'info',
+    message: `event ${n}`,
+    ...over,
+  };
+}
+
+// Wait until the write queue is fully drained and no trim is in flight
+async function settle(store, ms = 50) {
+  if (store.writeQueue.length === 0 && !store.writing && !store._trimScheduled) return;
+  await new Promise((r) => setTimeout(r, ms));
+  return settle(store, ms);
+}
+
+describe('DC-116: query().total is the true match count, not the page size', () => {
+  test('total reflects all matching events beyond limit/offset', async () => {
+    const store = makeStore({ maxMemory: 10000 });
+    for (let i = 0; i < 250; i++) store.append(ev(i));
+    await settle(store);
+
+    // Page of 10 — total must be 250, not 10
+    const r1 = store.query({ limit: 10 });
+    expect(r1.events).toHaveLength(10);
+    expect(r1.total).toBe(250);
+
+    // Same through pagination
+    const r2 = store.query({ limit: 100, offset: 200 });
+    expect(r2.events).toHaveLength(50);
+    expect(r2.total).toBe(250);
+
+    // Filters count matches beyond the page too
+    const r3 = store.query({ limit: 5, actor: 'actor-1' });
+    expect(r3.total).toBe(50);
+    expect(r3.events.every((e) => e.actor === 'actor-1')).toBe(true);
+  });
+
+  test('pages are disjoint and newest-first across offsets (dashboard pagination)', async () => {
+    const store = makeStore({ maxMemory: 10000 });
+    for (let i = 0; i < 30; i++) store.append(ev(i));
+    await settle(store);
+
+    const p1 = store.query({ limit: 10, offset: 0 }).events;
+    const p2 = store.query({ limit: 10, offset: 10 }).events;
+    const p3 = store.query({ limit: 10, offset: 20 }).events;
+    const ids = [...p1, ...p2, ...p3].map((e) => e.id);
+    expect(ids).toHaveLength(30);
+    expect(new Set(ids).size).toBe(30); // no overlap, no loss
+    // Newest first: event 29 (appended last) leads page 1
+    expect(p1[0].message).toBe('event 29');
+    expect(p3[9].message).toBe('event 0');
+  });
+});
+
+describe('DC-116: byte-budget trim always converges below the trigger', () => {
+  test('trims when byte budget exceeded even under the line cap (old code no-oped)', async () => {
+    // Fat lines (~600B each): 40 lines = ~24KB > 16KB budget, but well under
+    // any line cap. Pre-DC-116, _trim() returned early (lines <= maxDisk)
+    // while _maybeTrim kept firing.
+    const store = makeStore({ maxDisk: 1000, trimSizeLimit: 16 * 1024 });
+    const fat = 'x'.repeat(600);
+    for (let i = 0; i < 40; i++) store.append(ev(i, { message: fat }));
+    await settle(store, 100);
+
+    const size = fs.statSync(store.filePath).size;
+    expect(size).toBeLessThan(16 * 1024); // under the trigger
+    // The most recent events survived the trim
+    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
+    expect(lines.length).toBeGreaterThan(0);
+    expect(lines.length).toBeLessThanOrEqual(40);
+    const last = JSON.parse(lines[lines.length - 1]);
+    expect(last.target).toBe('t-39');
+  });
+
+  test('respects the line cap when lines are thin (maxDisk still honored)', async () => {
+    // Thin lines (~120B): 300 lines = ~36KB > 16KB budget; maxDisk=100 must
+    // cap retained lines at 100 (~12KB) — under budget either way.
+    const store = makeStore({ maxDisk: 100, trimSizeLimit: 16 * 1024 });
+    for (let i = 0; i < 300; i++) store.append(ev(i));
+    await settle(store, 100);
+
+    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
+    expect(lines.length).toBeLessThanOrEqual(100);
+    expect(fs.statSync(store.filePath).size).toBeLessThan(16 * 1024);
+    const last = JSON.parse(lines[lines.length - 1]);
+    expect(last.target).toBe('t-299');
+  });
+
+  test('byte ceiling drops oldest lines even when under the line cap (both constraints reconcile)', async () => {
+    // maxDisk=1000 (no line pressure) but budget forces byte reduction:
+    // 40 fat lines ~24KB -> must fall under 80% of 16KB = 12.8KB (~21 lines)
+    const store = makeStore({ maxDisk: 1000, trimSizeLimit: 16 * 1024 });
+    const fat = 'x'.repeat(600);
+    for (let i = 0; i < 40; i++) store.append(ev(i, { message: fat }));
+    await settle(store, 100);
+
+    const size = fs.statSync(store.filePath).size;
+    expect(size).toBeLessThanOrEqual(Math.floor(16 * 1024 * 0.8) + 700); // ceiling + one fat line
+    expect(size).toBeLessThan(16 * 1024);
+  });
+});
+
+describe('DC-116: trim/append race — events appended around a trim are never lost', () => {
+  test('appends landing during trim survive (write lock serializes trim vs append)', async () => {
+    const store = makeStore({ maxDisk: 50, trimSizeLimit: 8 * 1024 });
+    const fat = 'x'.repeat(400);
+    // Push past the byte budget so the NEXT idle write path triggers a trim
+    for (let i = 0; i < 20; i++) store.append(ev(i, { message: fat }));
+    await settle(store, 100);
+
+    // Rapid-fire appends around trims: each burst re-crosses the 8KB budget,
+    // forcing multiple trims while appends keep flowing. Budget sized so the
+    // FINAL burst (~3.3KB) always fits under the post-trim ceiling — the
+    // retention contract guarantees the newest burst survives intact.
+    const ids = [];
+    for (let round = 0; round < 5; round++) {
+      for (let i = 0; i < 6; i++) {
+        const stored = store.append(ev(100 + round * 6 + i, { message: fat }));
+        ids.push(stored.id);
+      }
+      await settle(store, 100);
+    }
+
+    // Every appended event must be either on disk or accounted for by the
+    // explicit retention caps (maxDisk=50 lines / 8KB byte budget). The last
+    // burst MUST be fully on disk (it fits the budget; nothing newer exists).
+    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
+    const diskIds = new Set(lines.map((l) => JSON.parse(l).id));
+    const lastBurst = ids.slice(-6);
+    for (const id of lastBurst) {
+      expect(diskIds.has(id)).toBe(true);
+    }
+    // And the file is back under budget
+    expect(fs.statSync(store.filePath).size).toBeLessThan(8 * 1024);
+  });
+
+  test('in-memory index stays queryable and consistent right after a trim', async () => {
+    const store = makeStore({ maxDisk: 10, trimSizeLimit: 8 * 1024 });
+    for (let i = 0; i < 60; i++) store.append(ev(i, { message: 'y'.repeat(300) }));
+    await settle(store, 150);
+
+    // Disk kept <=10 lines; memory still serves the capped window
+    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
+    expect(lines.length).toBeLessThanOrEqual(10);
+    const q = store.query({ limit: 5 });
+    expect(q.total).toBe(store.size());
+    expect(q.events).toHaveLength(5);
+  });
+});
+
+describe('DC-116: trim error paths release the write lock (no wedged store)', () => {
+  test('rename failure resets _trimScheduled and writing so later appends flow', async () => {
+    const store = makeStore({ maxDisk: 5, trimSizeLimit: 2 * 1024 });
+    const fat = 'x'.repeat(500);
+    for (let i = 0; i < 10; i++) store.append(ev(i, { message: fat }));
+    await settle(store, 100);
+
+    // Sabotage: make the tmp path unwritable so writeFile inside _trim fails
+    const tmpPath = store.filePath + '.tmp';
+    fs.mkdirSync(tmpPath); // a DIRECTORY at the tmp path breaks writeFile
+
+    for (let i = 10; i < 16; i++) store.append(ev(i, { message: fat }));
+    await settle(store, 200);
+
+    // Lock must be released despite the failure
+    expect(store.writing).toBe(false);
+    expect(store._trimScheduled).toBe(false);
+
+    fs.rmSync(tmpPath, { recursive: true, force: true });
+    // Appends still land on disk after the sabotage is cleared (write path
+    // was never wedged). The post-append idle trim may legitimately SHRINK
+    // the file back under budget, so assert on content, not size.
+    const last = store.append(ev(99, { message: fat }));
+    await settle(store, 100);
+    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
+    const diskIds = new Set(lines.map((l) => JSON.parse(l).id));
+    expect(diskIds.has(last.id)).toBe(true);
+  });
+});