dashcaddy/dashcaddy-api/__tests__/event-store-retention-dc116.test.js

/**
 * DC-116 regression pins — security event store retention + query.total.
 *
 * Background (2026-08-23, one day after DC-113 activated the caddy source):
 * live store had 46,494 events (16.5MB) growing ~2MB/day. Cold review of
 * src/security/event-store.js found three defects:
 *
 *   1. query().total lied: the scan broke at offset+limit, so `total` was
 *      capped at the page size (<=1000). LIVE user-facing impact — the
 *      dashboard "N events (24h)" stat (status/js/security-center.js reads
 *      data.total) and GET /hosts/:id/health events_24h showed 1000 when
 *      the real 24h count was tens of thousands.
 *   2. Trim trigger/curer mismatch: trigger was byte-based (>50MB) but the
 *      curer was line-count-based (no-op unless >maxDisk=100k lines). If the
 *      average line ever exceeded ~524B (50MB/100k — 0.5% of live lines were
 *      already >524B, scanner bursts inflate metadata), trim fired on every
 *      append and rewrote nothing — unbounded file + full-file re-read on
 *      the write path.
 *   3. Trim/append race: trim renamed over the file with appends in flight;
 *      events appended after trim's readFile landed on the unlinked inode
 *      and were silently lost.
 *
 * Tests use the REAL store with temp files. No mocks of the module under test.
 */

const path = require('path');
const fs = require('fs');
const os = require('os');

// Hermetic sinks (same pattern as caddy-worker-pipeline-dc113.test.js)
const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc116-store-'));
process.env.SECURITY_EVENT_LOG_FILE = path.join(TMP_DIR, 'security-events.jsonl');

const { SecurityEventStore } = require('../src/security/event-store');

const silence = { info: () => {}, warn: () => {}, error: () => {} };

function makeStore(opts = {}) {
  return new SecurityEventStore({
    log: silence,
    filePath: path.join(TMP_DIR, `store-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`),
    ...opts,
  });
}

// Deterministic event factory. `target` carries the unique marker — it is
// never overridden by the fat-payload tests, which replace `message`.
function ev(n, over = {}) {
  return {
    source_type: 'api',
    actor: `actor-${n % 5}`,
    action: `action-${n % 3}`,
    target: `t-${n}`,
    outcome: 'success',
    severity: 'info',
    message: `event ${n}`,
    ...over,
  };
}

// Wait until the write queue is fully drained and no trim is in flight
async function settle(store, ms = 50) {
  if (store.writeQueue.length === 0 && !store.writing && !store._trimScheduled) return;
  await new Promise((r) => setTimeout(r, ms));
  return settle(store, ms);
}

describe('DC-116: query().total is the true match count, not the page size', () => {
  test('total reflects all matching events beyond limit/offset', async () => {
    const store = makeStore({ maxMemory: 10000 });
    for (let i = 0; i < 250; i++) store.append(ev(i));
    await settle(store);

    // Page of 10 — total must be 250, not 10
    const r1 = store.query({ limit: 10 });
    expect(r1.events).toHaveLength(10);
    expect(r1.total).toBe(250);

    // Same through pagination
    const r2 = store.query({ limit: 100, offset: 200 });
    expect(r2.events).toHaveLength(50);
    expect(r2.total).toBe(250);

    // Filters count matches beyond the page too
    const r3 = store.query({ limit: 5, actor: 'actor-1' });
    expect(r3.total).toBe(50);
    expect(r3.events.every((e) => e.actor === 'actor-1')).toBe(true);
  });

  test('pages are disjoint and newest-first across offsets (dashboard pagination)', async () => {
    const store = makeStore({ maxMemory: 10000 });
    for (let i = 0; i < 30; i++) store.append(ev(i));
    await settle(store);

    const p1 = store.query({ limit: 10, offset: 0 }).events;
    const p2 = store.query({ limit: 10, offset: 10 }).events;
    const p3 = store.query({ limit: 10, offset: 20 }).events;
    const ids = [...p1, ...p2, ...p3].map((e) => e.id);
    expect(ids).toHaveLength(30);
    expect(new Set(ids).size).toBe(30); // no overlap, no loss
    // Newest first: event 29 (appended last) leads page 1
    expect(p1[0].message).toBe('event 29');
    expect(p3[9].message).toBe('event 0');
  });
});

describe('DC-116: byte-budget trim always converges below the trigger', () => {
  test('trims when byte budget exceeded even under the line cap (old code no-oped)', async () => {
    // Fat lines (~600B each): 40 lines = ~24KB > 16KB budget, but well under
    // any line cap. Pre-DC-116, _trim() returned early (lines <= maxDisk)
    // while _maybeTrim kept firing.
    const store = makeStore({ maxDisk: 1000, trimSizeLimit: 16 * 1024 });
    const fat = 'x'.repeat(600);
    for (let i = 0; i < 40; i++) store.append(ev(i, { message: fat }));
    await settle(store, 100);

    const size = fs.statSync(store.filePath).size;
    expect(size).toBeLessThan(16 * 1024); // under the trigger
    // The most recent events survived the trim
    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
    expect(lines.length).toBeGreaterThan(0);
    expect(lines.length).toBeLessThanOrEqual(40);
    const last = JSON.parse(lines[lines.length - 1]);
    expect(last.target).toBe('t-39');
  });

  test('respects the line cap when lines are thin (maxDisk still honored)', async () => {
    // Thin lines (~120B): 300 lines = ~36KB > 16KB budget; maxDisk=100 must
    // cap retained lines at 100 (~12KB) — under budget either way.
    const store = makeStore({ maxDisk: 100, trimSizeLimit: 16 * 1024 });
    for (let i = 0; i < 300; i++) store.append(ev(i));
    await settle(store, 100);

    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
    expect(lines.length).toBeLessThanOrEqual(100);
    expect(fs.statSync(store.filePath).size).toBeLessThan(16 * 1024);
    const last = JSON.parse(lines[lines.length - 1]);
    expect(last.target).toBe('t-299');
  });

  test('byte ceiling drops oldest lines even when under the line cap (both constraints reconcile)', async () => {
    // maxDisk=1000 (no line pressure) but budget forces byte reduction:
    // 40 fat lines ~24KB -> must fall under 80% of 16KB = 12.8KB (~21 lines)
    const store = makeStore({ maxDisk: 1000, trimSizeLimit: 16 * 1024 });
    const fat = 'x'.repeat(600);
    for (let i = 0; i < 40; i++) store.append(ev(i, { message: fat }));
    await settle(store, 100);

    const size = fs.statSync(store.filePath).size;
    expect(size).toBeLessThanOrEqual(Math.floor(16 * 1024 * 0.8) + 700); // ceiling + one fat line
    expect(size).toBeLessThan(16 * 1024);
  });
});

describe('DC-116: trim/append race — events appended around a trim are never lost', () => {
  test('appends landing during trim survive (write lock serializes trim vs append)', async () => {
    const store = makeStore({ maxDisk: 50, trimSizeLimit: 8 * 1024 });
    const fat = 'x'.repeat(400);
    // Push past the byte budget so the NEXT idle write path triggers a trim
    for (let i = 0; i < 20; i++) store.append(ev(i, { message: fat }));
    await settle(store, 100);

    // Rapid-fire appends around trims: each burst re-crosses the 8KB budget,
    // forcing multiple trims while appends keep flowing. Budget sized so the
    // FINAL burst (~3.3KB) always fits under the post-trim ceiling — the
    // retention contract guarantees the newest burst survives intact.
    const ids = [];
    for (let round = 0; round < 5; round++) {
      for (let i = 0; i < 6; i++) {
        const stored = store.append(ev(100 + round * 6 + i, { message: fat }));
        ids.push(stored.id);
      }
      await settle(store, 100);
    }

    // Every appended event must be either on disk or accounted for by the
    // explicit retention caps (maxDisk=50 lines / 8KB byte budget). The last
    // burst MUST be fully on disk (it fits the budget; nothing newer exists).
    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
    const diskIds = new Set(lines.map((l) => JSON.parse(l).id));
    const lastBurst = ids.slice(-6);
    for (const id of lastBurst) {
      expect(diskIds.has(id)).toBe(true);
    }
    // And the file is back under budget
    expect(fs.statSync(store.filePath).size).toBeLessThan(8 * 1024);
  });

  test('in-memory index stays queryable and consistent right after a trim', async () => {
    const store = makeStore({ maxDisk: 10, trimSizeLimit: 8 * 1024 });
    for (let i = 0; i < 60; i++) store.append(ev(i, { message: 'y'.repeat(300) }));
    await settle(store, 150);

    // Disk kept <=10 lines; memory still serves the capped window
    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
    expect(lines.length).toBeLessThanOrEqual(10);
    const q = store.query({ limit: 5 });
    expect(q.total).toBe(store.size());
    expect(q.events).toHaveLength(5);
  });
});

describe('DC-116: trim error paths release the write lock (no wedged store)', () => {
  test('rename failure resets _trimScheduled and writing so later appends flow', async () => {
    const store = makeStore({ maxDisk: 5, trimSizeLimit: 2 * 1024 });
    const fat = 'x'.repeat(500);
    for (let i = 0; i < 10; i++) store.append(ev(i, { message: fat }));
    await settle(store, 100);

    // Sabotage: make the tmp path unwritable so writeFile inside _trim fails
    const tmpPath = store.filePath + '.tmp';
    fs.mkdirSync(tmpPath); // a DIRECTORY at the tmp path breaks writeFile

    for (let i = 10; i < 16; i++) store.append(ev(i, { message: fat }));
    await settle(store, 200);

    // Lock must be released despite the failure
    expect(store.writing).toBe(false);
    expect(store._trimScheduled).toBe(false);

    fs.rmSync(tmpPath, { recursive: true, force: true });
    // Appends still land on disk after the sabotage is cleared (write path
    // was never wedged). The post-append idle trim may legitimately SHRINK
    // the file back under budget, so assert on content, not size.
    const last = store.append(ev(99, { message: fat }));
    await settle(store, 100);
    const lines = fs.readFileSync(store.filePath, 'utf8').trim().split('\n');
    const diskIds = new Set(lines.map((l) => JSON.parse(l).id));
    expect(diskIds.has(last.id)).toBe(true);
  });
});