fix(security): caddy-event self-noise conjunction filter for host curl probes (DC-118) [glm-grade=A]

The host-side uptime watchdog and on-host cron jobs curl Caddy with a stock curl/8.5.0 UA from loopback and the host tailscale IP — ~300 GET /api/health 401 warn-events/day burying real perimeter signal (census: 339+20 of 5000 access.log lines). Dropping on UA alone would blind the store to external curl scanners, so generic tool UAs (curl/) are now dropped ONLY when the source remote_ip is one of this host's own addresses (DASHCADDY_SELF_IPS, default loopback; start.sh derives 127.0.0.1 + tailscale ip -4, empty-safe). remote_ip (TCP peer) is used, never the spoofable client_ip. DashCaddy-* probe UAs stay unconditionally dropped. 9 regression pins cover the full conjunction matrix incl. external-IP+curl KEPT and spoofed-XFF KEPT. 2858/2858 green (128 suites). Judge: glm-4.6@zai-coding-paas cold-read round 1 = A clean (0 blocking), both polish notes folded. URN urn:ump:s2sgitfepze65crtp57dpdw4gk4w7upsoi4tqcvfwahcsicyepba
2026-08-23 15:53:58 -07:00
parent 2dce6dca5e
commit 1744d1c86e
3 changed files with 214 additions and 3 deletions
@@ -0,0 +1,183 @@
+/**
+ * DC-118 regression pins — generic-UA self-noise conjunction filter.
+ *
+ * Live census (2026-08-23, /var/log/caddy/access.log): the DNS2 watchdog
+ * and on-host cron jobs hit Caddy with a stock curl/8.5.0 UA from
+ * 127.0.0.1 (339/5000 lines) and the host's own tailscale IP (20/5000) —
+ * ~300 GET /api/health 401 warn-events/day burying real perimeter
+ * signal. External curl traffic (zgrab/ scanners using curl, real
+ * attackers) MUST stay visible.
+ *
+ * Design: DashCaddy-* probe UA prefixes are dropped unconditionally
+ * (they are our own binaries). GENERIC tool UAs (curl/) are dropped ONLY
+ * when the source remote_ip is one of this host's own addresses
+ * (DASHCADDY_SELF_IPS env, default loopback). remote_ip (the TCP peer)
+ * is the input — never client_ip/X-Forwarded-For, which is spoofable.
+ *
+ * All tests use the REAL worker: temp access log, real tail, real event
+ * store, hermetic sinks. No mocks of the module under test.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+
+// Hermetic sinks (same pattern as caddy-worker-pipeline-dc113.test.js)
+const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'dc118-caddy-'));
+process.env.SECURITY_EVENT_LOG_FILE = path.join(TMP_DIR, 'security-events.jsonl');
+process.env.CADDY_ACCESS_LOG = path.join(TMP_DIR, 'access.log');
+process.env.DATA_DIR = TMP_DIR;
+// Self-IP set for these tests: loopback defaults + a fake tailscale IP.
+process.env.DASHCADDY_SELF_IPS = '127.0.0.1,::1,100.121.150.22';
+
+const { startCaddyWorker } = require('../src/security/event-workers');
+const { getStore } = require('../src/security/event-store');
+
+let capturedWarns = [];
+let capturedInfos = [];
+const fakeLogger = {
+  warn: (ctx, msg, extra) => capturedWarns.push({ ctx, msg, extra }),
+  info: (ctx, msg, extra) => capturedInfos.push({ ctx, msg, extra }),
+  error: () => {},
+};
+
+const ACCESS_LOG = process.env.CADDY_ACCESS_LOG;
+const STORE_FILE = process.env.SECURITY_EVENT_LOG_FILE;
+
+function mkLine({ ip, ua, uri = '/api/health', status = 401 }) {
+  return JSON.stringify({
+    ts: Date.now() / 1000,
+    request: {
+      remote_ip: ip, method: 'GET', uri, host: 'status.sami', proto: 'HTTP/2.0',
+      headers: ua === null ? {} : { 'User-Agent': [ua] },
+    },
+    status,
+    duration: 0.004,
+  }) + '\n';
+}
+
+function readStored() {
+  try {
+    return fs.readFileSync(STORE_FILE, 'utf8').trim().split('\n')
+      .filter(Boolean).map(l => JSON.parse(l));
+  } catch { return []; }
+}
+
+async function waitForEvents(n, { timeoutMs = 5000 } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const events = readStored().filter(e => e.source_type === 'caddy');
+    if (events.length >= n) return events;
+    await new Promise(r => setTimeout(r, 50));
+  }
+  throw new Error(`timed out waiting for ${n} caddy events (got ${readStored().filter(e => e.source_type === 'caddy').length})`);
+}
+
+async function waitForQuiet({ settleMs = 1200 } = {}) {
+  // Inverse of waitForEvents: give the tail a window to (wrongly) emit,
+  // then assert it did not.
+  await new Promise(r => setTimeout(r, settleMs));
+  return readStored().filter(e => e.source_type === 'caddy');
+}
+
+beforeEach(() => {
+  fs.writeFileSync(STORE_FILE, '', 'utf8');
+  fs.writeFileSync(ACCESS_LOG, '', 'utf8');
+  // Reset the tail's persisted offset (same flake lesson as DC-112/113).
+  fs.writeFileSync(path.join(TMP_DIR, '.caddy-tail-offset'), '0', 'utf8');
+  capturedWarns = [];
+  capturedInfos = [];
+  getStore({ log: fakeLogger }); // fresh singleton pointed at the temp store file
+});
+
+afterAll(() => {
+  try { fs.rmSync(TMP_DIR, { recursive: true, force: true }); } catch {}
+});
+
+describe('DC-118: generic-UA self-noise conjunction filter', () => {
+  let worker;
+  afterEach(() => { if (worker) { worker.stop(); worker = null; } });
+
+  test('matrix cell 1 — self IP + generic curl UA → DROPPED (loopback watchdog)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '127.0.0.1', ua: 'curl/8.5.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const events = await waitForQuiet();
+    expect(events.length).toBe(0);
+  });
+
+  test('matrix cell 1b — self tailscale IP + curl UA → DROPPED (on-host cron)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '100.121.150.22', ua: 'curl/8.5.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const events = await waitForQuiet();
+    expect(events.length).toBe(0);
+  });
+
+  test('matrix cell 2 — EXTERNAL IP + curl UA → KEPT (real attacker visibility)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '198.51.100.7', ua: 'curl/8.5.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const [ev] = await waitForEvents(1);
+    expect(ev.actor).toBe('198.51.100.7');
+    expect(ev.metadata.user_agent).toBe('curl/8.5.0');
+    expect(ev.action).toBe('http.401');
+    expect(ev.severity).toBe('warn'); // /api/health 401 stays a warn-event
+  });
+
+  test('matrix cell 3 — self IP + NON-generic UA (browser/attacker tool) → KEPT', async () => {
+    // Even from our own IP, a browser or attack tool UA must not be
+    // silently discarded — an attacker landing on the host itself is
+    // exactly the event the store exists to keep.
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '127.0.0.1', ua: 'Mozilla/5.0 zgrab/0.x' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const [ev] = await waitForEvents(1);
+    expect(ev.actor).toBe('127.0.0.1');
+    expect(ev.metadata.user_agent).toBe('Mozilla/5.0 zgrab/0.x');
+  });
+
+  test('matrix cell 4 — self IP + no UA at all → KEPT (missing UA is not noise)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '127.0.0.1', ua: null }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const [ev] = await waitForEvents(1);
+    expect(ev.actor).toBe('127.0.0.1');
+    expect(ev.metadata.user_agent).toBeNull();
+  });
+
+  test('spoofed X-Forwarded-For (client_ip) cannot opt an attacker out — filter reads remote_ip only', async () => {
+    fs.appendFileSync(ACCESS_LOG, JSON.stringify({
+      ts: Date.now() / 1000,
+      request: {
+        remote_ip: '198.51.100.9', client_ip: '127.0.0.1', // claims to be us
+        method: 'GET', uri: '/api/health', host: 'status.sami', proto: 'HTTP/2.0',
+        headers: { 'User-Agent': ['curl/8.5.0'] },
+      },
+      status: 401,
+      duration: 0.004,
+    }) + '\n');
+    worker = startCaddyWorker({ log: fakeLogger });
+    const [ev] = await waitForEvents(1);
+    expect(ev.actor).toBe('198.51.100.9'); // TCP peer, not the spoofable header
+  });
+
+  test('IPv6 loopback ::1 with curl UA → DROPPED (env-listed self IP)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '::1', ua: 'curl/8.5.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const events = await waitForQuiet();
+    expect(events.length).toBe(0);
+  });
+
+  test('DashCaddy-* probe UA from a NON-self IP is still dropped (own binaries, unconditional)', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '172.17.0.4', ua: 'DashCaddy-HealthCheck/1.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    const events = await waitForQuiet();
+    expect(events.length).toBe(0);
+  });
+
+  test('prefix future-proofing: curl/10.0 from self IP → DROPPED; curl-impersonate NOT dropped', async () => {
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '127.0.0.1', ua: 'curl/10.0' }));
+    fs.appendFileSync(ACCESS_LOG, mkLine({ ip: '198.51.100.10', ua: 'curl-impersonate-chrome/1.0' }));
+    worker = startCaddyWorker({ log: fakeLogger });
+    // curl-impersonate does not match the 'curl/' prefix; kept from any IP.
+    const events = await waitForEvents(1);
+    expect(events.length).toBe(1);
+    expect(events[0].metadata.user_agent).toBe('curl-impersonate-chrome/1.0');
+  });
+});