dashcaddy/dashcaddy-api/src/monitoring/caddy-upstream-watcher.js

/**
 * Caddy upstream watcher
 *
 * Watches every `reverse_proxy <host>` directive in /etc/caddy/sites/* and
 * independently probes each upstream every 60s. After 5 minutes of
 * consecutive failures, emits a `caddy-upstream-dead` incident via the shared
 * healthChecker so the dashboard can surface it.
 *
 * This is intentionally separate from Caddy's own `reverse_proxy` health
 * checker: Caddy probes log every failure to syslog (the noisy spam the
 * dashboard currently sees for `100.120.159.34:5000`), but Caddy never
 * surfaces the result to the dashboard or to the API. This watcher gives
 * the operator (a) a deduped view, (b) a 5-minute confirmation window so a
 * one-off blip doesn't page, and (c) a mute toggle to silence known-dead
 * upstreams without editing the Caddyfile.
 *
 * State persisted to <dataDir>/caddy-upstreams.json. Mute list is part of
 * the same file so atomic-write semantics keep state + mutes consistent.
 *
 * The probe DOES NOT use Caddy's health_uri (that's Caddy's own probe and
 * the source of the spam). The probe also stamps `X-DashCaddy-HealthCheck: 1`
 * so the `dashcaddy_auth` forward_auth gate on *.sami bypasses for probes
 * (same trick as src/monitoring/health-checker.js _doRequest).
 *
 * @module caddy-upstream-watcher
 */

const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');
const EventEmitter = require('events');
const platformPaths = require('../../platform-paths');

/** Default probe interval: 60s. Independent of Caddy's 10s internal probe. */
const PROBE_INTERVAL_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_INTERVAL_MS || '60000', 10);

/** Per-probe timeout. Short — these are liveness pings, not full requests. */
const PROBE_TIMEOUT_MS = parseInt(process.env.CADDY_UPSTREAM_PROBE_TIMEOUT_MS || '5000', 10);

/** After this many ms of continuous failure, emit a "dead" incident. */
const DEAD_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_DEAD_AFTER_MS || (5 * 60 * 1000), 10);

/** After this many ms of continuous success, auto-resolve any open incident. */
const RESOLVED_AFTER_MS = parseInt(process.env.CADDY_UPSTREAM_RESOLVED_AFTER_MS || (60 * 1000), 10);

/** Status codes that prove the upstream answered. 4xx auth-walled counts as up. */
const HEALTHY_CODES = new Set([200, 201, 204, 301, 302, 303, 307, 308, 401, 403, 429]);

const STATE_FILE = process.env.CADDY_UPSTREAMS_STATE_FILE
  || path.join(platformPaths.dataDir || path.dirname(platformPaths.configFile || '.'), 'caddy-upstreams.json');

const SITES_DIR = process.env.CADDY_SITES_DIR || '/etc/caddy/sites';

/**
 * Hostname the probe uses instead of a loopback address.
 *
 * CRITICAL: this watcher runs INSIDE the dashcaddy-api container. Caddy runs
 * on the HOST. A site config's `reverse_proxy localhost:8088` means "the
 * host's loopback" from Caddy's point of view — but from inside the container
 * `localhost`/`127.0.0.1` is the container's OWN loopback, where nothing
 * listens. Probing loopback verbatim makes every healthy host-side upstream
 * report ECONNREFUSED (live prod bug 2026-08-18: 9 of 14 tracked upstreams
 * showed 278 consecutive phantom failures and opened bogus `caddy-upstream-dead`
 * incidents).
 *
 * Fix: remap loopback probe targets to `host.docker.internal`, which start.sh
 * pins to the host's bridge IP via `--add-host=host.docker.internal:host-gateway`
 * (Docker ≥ 20.10). The upstream's display key stays `localhost:PORT` so
 * existing mute lists and UI labels are unaffected — only the probe target
 * changes. Set IN_CONTAINER=false (e.g. a bare-metal deployment where the API
 * runs beside Caddy) to disable the remap.
 */
const HOST_GATEWAY_NAME = process.env.CADDY_UPSTREAM_HOST_GATEWAY_NAME || 'host.docker.internal';
const IN_CONTAINER = process.env.IN_CONTAINER !== 'false';
const HOST_GATEWAY_PROBE = IN_CONTAINER ? HOST_GATEWAY_NAME : null;

/** True when the address is IPv4 loopback (127.0.0.0/8) or the `localhost` name. */
function isLoopbackHost(host) {
  return host === 'localhost' || /^127(\.\d{1,3}){3}$/.test(host);
}

class CaddyUpstreamWatcher extends EventEmitter {
  constructor(opts = {}) {
    super();
    this.log = opts.log || console;
    this.healthChecker = opts.healthChecker || null;
    /** Map<string, UpstreamState> keyed by host (host[:port]) */
    this.upstreams = new Map();
    /** Set<string> hosts the user has muted */
    this.muted = new Set();
    /** Set<string> incident IDs currently open — prevents duplicate incidents */
    this.openIncidents = new Set();
    this.timer = null;
    this.checking = false;
    this.scanTimer = null;
    this._loadState();
  }

  /** Begin watching. Idempotent — safe to call twice. */
  start() {
    if (this.checking) return;
    this.checking = true;
    // Initial scan + probe so the dashboard has data immediately after boot.
    this.scanSites().catch((e) => this.log.warn('caddy-upstream-watcher', e?.message || String(e)));
    this.timer = setInterval(() => this._tick().catch(() => {}), PROBE_INTERVAL_MS);
    // Re-scan sites every 5 min so newly added sites get picked up.
    this.scanTimer = setInterval(() => this.scanSites().catch(() => {}), 5 * 60 * 1000);
    this.log.info?.('caddy-upstream-watcher', 'started', {
      probeIntervalMs: PROBE_INTERVAL_MS,
      deadAfterMs: DEAD_AFTER_MS,
      stateFile: STATE_FILE,
      sitesDir: SITES_DIR
    }) ?? this.log.info?.('caddy-upstream-watcher', 'started');
  }

  stop() {
    if (!this.checking) return;
    this.checking = false;
    if (this.timer) clearInterval(this.timer);
    if (this.scanTimer) clearInterval(this.scanTimer);
    this.timer = null;
    this.scanTimer = null;
  }

  /** Parse /etc/caddy/sites/* and seed/refresh the upstream map. */
  async scanSites() {
    let entries;
    try {
      entries = fs.readdirSync(SITES_DIR);
    } catch (e) {
      // Sites dir might not exist in dev — that's OK, just skip.
      this.log.warn?.('caddy-upstream-watcher', `cannot read ${SITES_DIR}: ${e.message}`);
      return;
    }

    const seen = new Set();
    for (const entry of entries) {
      // Caddy `import` sites have a wild mix of extensions: `.sami`,
      // `.caddy`, `.conf` — and ALSO bare hostnames like
      // `zap.sami-ahmed.net`, `samitest.space`, `blocks.cryptographic-triangles.org`
      // where the "extension" is `.net`/`.space`/`.org`. Filter out known
      // non-site junk (readmes, .bak) and accept everything else; the
      // reverse_proxy parse below is the real validation.
      if (/^README|\.bak$|\.swp$|^\.|^#/.test(entry)) continue;
      if (entry === 'Caddyfile' || entry === 'caddyfile') continue;
      const filePath = path.join(SITES_DIR, entry);
      let content;
      try {
        content = fs.readFileSync(filePath, 'utf8');
      } catch (_) { continue; }

      // Cheap pre-check: skip files with no reverse_proxy and no brace block
      // (README files, .gitignore, etc.). The reverse_proxy regex below is
      // the authoritative parse, but this avoids regex-scanning every
      // unrelated file in the directory.
      if (!/reverse_proxy/i.test(content)) continue;

      // Capture the site block host from the first line: e.g. "arch.sami {"
      const siteMatch = content.match(/^\s*([a-z0-9._-]+)\s*\{/im);
      const siteName = siteMatch ? siteMatch[1] : entry.replace(/\.(sami|caddy|conf)$/i, '');

      // Find every reverse_proxy <host[:port]> directive. Match common shapes:
      //   reverse_proxy 100.120.159.34:5000 { ... }
      //   reverse_proxy http://100.120.159.34:5000 { ... }
      //   reverse_proxy 100.120.159.34:5000
      const re = /reverse_proxy\s+(?:https?:\/\/)?([0-9]{1,3}(?:\.[0-9]{1,3}){3}|[a-z0-9._-]+)(?::(\d+))?/gi;
      let m;
      while ((m = re.exec(content)) !== null) {
        const host = m[1];
        let port = m[2];
        if (!port) {
          if (m[0].includes('https')) port = '443';
          else if (m[0].includes('http://')) port = '80';
          else port = '';
        }
        const key = port ? `${host}:${port}` : host;
        seen.add(key);
        if (!this.upstreams.has(key)) {
          this.upstreams.set(key, {
            host: key,
            ip: host,
            port: port || null,
            site: siteName,
            siteFile: entry,
            consecutiveFailures: 0,
            lastFailureAt: null,
            lastSuccessAt: null,
            lastError: null,
            lastCheckedAt: null,
            status: 'unknown'
          });
        } else {
          // Refresh site name/file in case the file was renamed.
          const u = this.upstreams.get(key);
          u.site = siteName;
          u.siteFile = entry;
        }
      }
    }

    // Drop upstreams that disappeared from the Caddyfile (removed/renamed site).
    for (const key of Array.from(this.upstreams.keys())) {
      if (!seen.has(key)) this.upstreams.delete(key);
    }

    this._saveState();
  }

  /** Single probe tick over every upstream. */
  async _tick() {
    const probes = [];
    for (const u of this.upstreams.values()) {
      if (this.muted.has(u.host)) continue;
      probes.push(this._probeOne(u).catch((e) => {
        this.log.warn?.('caddy-upstream-watcher', `probe failed for ${u.host}: ${e.message}`);
      }));
    }
    await Promise.all(probes);
    this._saveState();
    this.emit('tick', this.snapshot());
  }

  /** Probe a single upstream and update state. */
  async _probeOne(u) {
    // Loopback upstreams (see HOST_GATEWAY_PROBE header comment): the Caddyfile
    // `localhost`/`127.x` is host-relative, so probe the host gateway instead of
    // the container's own loopback. Display key and persisted `ip` are unchanged.
    const loopbackRemap = !!(HOST_GATEWAY_PROBE && isLoopbackHost(u.ip));
    const probeHost = loopbackRemap ? HOST_GATEWAY_PROBE : u.ip;
    const result = await this._doProbe(probeHost, u.port);
    u.lastCheckedAt = new Date().toISOString();

    if (result.healthy) {
      u.consecutiveFailures = 0;
      u.lastSuccessAt = u.lastCheckedAt;
      u.lastError = null;
      // Resolve open incident if upstream is healthy for RESOLVED_AFTER_MS.
      this._maybeResolve(u);
      // Only flip to 'up' if the upstream has been healthy long enough to not
      // be a flapping signal — short blips are normal and we want the dashboard
      // to be stable. After one full successful check we mark 'up' but the
      // incident resolution waits for RESOLVED_AFTER_MS.
      u.status = 'up';
      // A successful host-gateway probe PROVES the bridge can reach the
      // host. If a later probe then fails, we have strong evidence the
      // upstream itself went dead — not that bridge connectivity broke.
      // Mark verifiedViaBridge so the unverifiable path can short-circuit
      // and treat it like a non-loopback upstream.
      if (loopbackRemap) u.verifiedViaBridge = true;
    } else if (loopbackRemap && !u.verifiedViaBridge) {
      // The host-gateway probe comes from the docker bridge IP. A service
      // bound to 0.0.0.0 on the host answers; a service bound to the host's
      // 127.0.0.1 ONLY refuses — indistinguishable, from this vantage point,
      // from a truly dead service. Caddy (on the host) reaches both fine, so
      // a failed probe here is NOT evidence the upstream is dead. Mark it
      // unverifiable: no failure counters, no incident, keep lastError for
      // visibility. (A successful probe IS conclusive — see above.)
      u.consecutiveFailures = 0;
      u.status = 'unverifiable';
      u.lastError = `host-loopback upstream not verifiable from container (${result.error || `HTTP ${result.statusCode || 'unknown'}`})`;
      // Clear the success anchor: a 10-minute-old success is not evidence of
      // anything for an upstream we cannot observe from this vantage point,
      // and leaving it would make snapshot() compute a bogus failingForMs
      // and flag `dead`.
      u.lastSuccessAt = null;
      this._maybeResolve(u);
    } else if (loopbackRemap && u.verifiedViaBridge) {
      // The bridge previously reached this upstream successfully — so a
      // failed probe here is near-conclusive evidence the upstream itself
      // went dead (the bridge path itself doesn't change between probes).
      // Treat it like a non-loopback upstream failure: count it, open an
      // incident after DEAD_AFTER_MS. This restores dead-detection for the
      // subset of loopback upstreams that prove themselves reachable.
      u.consecutiveFailures += 1;
      u.lastFailureAt = u.lastCheckedAt;
      u.lastError = result.error || `HTTP ${result.statusCode || 'unknown'}`;
      u.status = 'down';
      this._maybeOpenIncident(u);
    } else {
      u.consecutiveFailures += 1;
      u.lastFailureAt = u.lastCheckedAt;
      u.lastError = result.error || `HTTP ${result.statusCode || 'unknown'}`;
      // First failure flips status to 'down' immediately for the dashboard, but
      // we only OPEN an incident after the upstream has been continuously failing
      // for DEAD_AFTER_MS (5 min by default) so a single transient blip doesn't
      // page anyone.
      u.status = 'down';
      this._maybeOpenIncident(u);
    }
  }

  _maybeOpenIncident(u) {
    if (!this.healthChecker) return;
    // "failingForMs" = continuous time the upstream has been unhealthy.
    // Use lastSuccessAt as the anchor — if it was up 7min ago and is still
    // down now, that's 7 minutes of continuous failure regardless of how many
    // individual probe failures have piled up in between. Falls back to
    // consecutiveFailures * interval when there's no success anchor (e.g. we've
    // never seen the upstream healthy since startup).
    const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
    const failingForMs = lastSuccessMs !== null
      ? Math.max(0, Date.now() - lastSuccessMs)
      : u.consecutiveFailures * PROBE_INTERVAL_MS;
    if (failingForMs < DEAD_AFTER_MS) return;
    if (this.openIncidents.has(u.host)) return;

    // Mimic the shape HealthChecker.createIncident expects.
    try {
      this.healthChecker.createIncident(u.host, 'caddy-upstream-dead',
        `Caddy upstream ${u.host} (site ${u.site}) unreachable for ${Math.round(failingForMs / 60000)}m: ${u.lastError || 'no response'}`,
        {
          serviceId: u.host,
          timestamp: u.lastFailureAt,
          status: 'down',
          error: u.lastError,
          details: { site: u.site, siteFile: u.siteFile }
        }
      );
      this.openIncidents.add(u.host);
      this.emit('upstream-dead', u);
      this.log.warn?.('caddy-upstream-watcher', `upstream dead: ${u.host} (${u.site})`);
    } catch (e) {
      this.log.warn?.('caddy-upstream-watcher', `incident create failed: ${e.message}`);
    }
  }

  _maybeResolve(u) {
    if (!this.healthChecker) return;
    if (!this.openIncidents.has(u.host)) return;
    const downSince = u.lastFailureAt ? new Date(u.lastFailureAt).getTime() : 0;
    const recoveredForMs = downSince ? Date.now() - downSince : 0;
    if (recoveredForMs < RESOLVED_AFTER_MS) return;
    try {
      this.healthChecker.resolveIncident(u.host, 'caddy-upstream-dead', {
        serviceId: u.host,
        timestamp: u.lastSuccessAt || new Date().toISOString(),
        status: 'up'
      });
      this.openIncidents.delete(u.host);
      this.emit('upstream-recovered', u);
      this.log.info?.('caddy-upstream-watcher', `upstream recovered: ${u.host}`);
    } catch (e) {
      this.log.warn?.('caddy-upstream-watcher', `incident resolve failed: ${e.message}`);
    }
  }

  _doProbe(host, port) {
    return new Promise((resolve) => {
      const isHttps = port === '443';
      const lib = isHttps ? https : http;
      const opts = {
        hostname: host,
        port: port || (isHttps ? 443 : 80),
        method: 'HEAD',
        path: '/',
        timeout: PROBE_TIMEOUT_MS,
        headers: { 'X-DashCaddy-HealthCheck': '1', 'User-Agent': 'DashCaddy-CaddyUpstreamWatcher/1' },
        rejectUnauthorized: false
      };
      const req = lib.request(opts, (res) => {
        res.resume();
        const healthy = HEALTHY_CODES.has(res.statusCode);
        resolve({ healthy, statusCode: res.statusCode });
      });
      req.on('timeout', () => {
        req.destroy(new Error('probe timeout'));
      });
      req.on('error', (err) => {
        resolve({ healthy: false, error: err.message });
      });
      req.end();
    });
  }

  /**
   * Public snapshot for the API/UI.
   *
   * Each upstream record includes:
   *  - host / site / siteFile: identity
   *  - status: 'up' | 'down' | 'unverifiable' | 'unknown' (or 'muted' here)
   *  - consecutiveFailures / failingForMs: dead-detection counters
   *  - lastCheckedAt / lastSuccessAt / lastFailureAt / lastError: probe history
   *  - muted: true if user silenced this upstream
   *  - dead: true if failingForMs >= DEAD_AFTER_MS (5 min default)
   *  - verifiedViaBridge (loopback upstreams only): true iff this upstream
   *    has ever answered a host-gateway probe with success. A later failed
   *    probe is then near-conclusive evidence of upstream death rather
   *    than bridge/UFW refusal. UI consumers should label `unverifiable`
   *    rows as "no prior observation" and `down` rows with
   *    verifiedViaBridge=true as "previously-verified, now down".
   *
   * @returns {{ upstreams: Array<object>, config: object }}
   */
  snapshot() {
    const list = [];
    for (const u of this.upstreams.values()) {
      const muted = this.muted.has(u.host);
      // Same anchor as _maybeOpenIncident: time since the last successful
      // probe. If we've never seen a success, fall back to consecutive
      // failures × probe interval as a worst-case lower bound.
      const lastSuccessMs = u.lastSuccessAt ? new Date(u.lastSuccessAt).getTime() : null;
      let failingFor = 0;
      if (!muted) {
        if (lastSuccessMs !== null) {
          failingFor = Math.max(0, Date.now() - lastSuccessMs);
        } else if (u.status === 'down') {
          failingFor = u.consecutiveFailures * PROBE_INTERVAL_MS;
        }
      }
      list.push({
        host: u.host,
        site: u.site,
        siteFile: u.siteFile,
        status: muted ? 'muted' : u.status,
        consecutiveFailures: u.consecutiveFailures,
        lastCheckedAt: u.lastCheckedAt,
        lastSuccessAt: u.lastSuccessAt,
        lastFailureAt: u.lastFailureAt,
        lastError: u.lastError,
        failingForMs: failingFor,
        muted,
        dead: !muted && failingFor >= DEAD_AFTER_MS,
        // True iff this loopback upstream has ever answered a host-gateway
        // probe with success — meaning we have at least one prior positive
        // observation of bridge connectivity, so a later failure is
        // evidence of upstream death rather than bridge/UFW refusal.
        verifiedViaBridge: !!u.verifiedViaBridge
      });
    }
    // Sort: dead first, then down, then muted, then unverifiable (informational),
    // then up, then unknown. Within each, by host.
    list.sort((a, b) => {
      const order = { dead: 0, down: 1, muted: 2, unverifiable: 3, up: 4, unknown: 5 };
      const oa = order[a.dead ? 'dead' : a.status] ?? 9;
      const ob = order[b.dead ? 'dead' : b.status] ?? 9;
      if (oa !== ob) return oa - ob;
      return a.host.localeCompare(b.host);
    });
    return {
      upstreams: list,
      config: {
        probeIntervalMs: PROBE_INTERVAL_MS,
        deadAfterMs: DEAD_AFTER_MS,
        resolvedAfterMs: RESOLVED_AFTER_MS,
        sitesDir: SITES_DIR
      }
    };
  }

  setMuted(host, muted) {
    if (muted) {
      this.muted.add(host);
    } else {
      this.muted.delete(host);
      // Reset failure state on unmute so we don't immediately re-incident a
      // upstream that just came off mute.
      const u = this.upstreams.get(host);
      if (u) {
        u.consecutiveFailures = 0;
        u.lastError = null;
        u.lastFailureAt = null;
        u.status = 'unknown';
      }
    }
    this._saveState();
    return { host, muted: !!muted };
  }

  isMuted(host) { return this.muted.has(host); }

  _loadState() {
    try {
      if (!fs.existsSync(STATE_FILE)) return;
      const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
      if (Array.isArray(data.muted)) this.muted = new Set(data.muted);
      // Don't reload upstreams from disk — sites dir is the source of truth.
      // But preserve last-check state for hosts that still exist.
      if (data.upstreams && typeof data.upstreams === 'object') {
        this._restoreUpstreamStates(data.upstreams);
      }
    } catch (e) {
      this.log.warn?.('caddy-upstream-watcher', `state load failed: ${e.message}`);
    }
  }

  _restoreUpstreamStates(persisted) {
    for (const [host, st] of Object.entries(persisted)) {
      if (this.upstreams.has(host)) continue;
      this.upstreams.set(host, {
        host,
        ip: st.ip || host.split(':')[0],
        port: st.port || null,
        site: st.site || '',
        siteFile: st.siteFile || '',
        consecutiveFailures: st.consecutiveFailures || 0,
        lastFailureAt: st.lastFailureAt || null,
        lastSuccessAt: st.lastSuccessAt || null,
        lastError: st.lastError || null,
        lastCheckedAt: st.lastCheckedAt || null,
        // Persist verifiedViaBridge so a loopback upstream that proved itself
        // reachable once doesn't have to re-prove it after every container
        // restart. A 1-tick blip is acceptable here because:
        //  (a) the field is only used as a labelling gate for the
        //      unverifiable-vs-down decision — a falsy restart value means
        //      we re-mark unverifiable for one cycle, the safer direction;
        //  (b) the bridge IP doesn't change between restarts of the same
        //      container, so a previously-positive observation is still
        //      good evidence.
        verifiedViaBridge: !!st.verifiedViaBridge,
        status: 'unknown'
      });
    }
  }

  _saveState() {
    try {
      const dir = path.dirname(STATE_FILE);
      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
      const upstreams = {};
      for (const [k, v] of this.upstreams.entries()) {
        upstreams[k] = {
          ip: v.ip,
          port: v.port,
          site: v.site,
          siteFile: v.siteFile,
          consecutiveFailures: v.consecutiveFailures,
          lastFailureAt: v.lastFailureAt,
          lastSuccessAt: v.lastSuccessAt,
          lastError: v.lastError,
          lastCheckedAt: v.lastCheckedAt,
          verifiedViaBridge: !!v.verifiedViaBridge
        };
      }
      const tmp = STATE_FILE + '.tmp';
      fs.writeFileSync(tmp, JSON.stringify({ muted: Array.from(this.muted), upstreams }, null, 2));
      fs.renameSync(tmp, STATE_FILE);
    } catch (e) {
      this.log.warn?.('caddy-upstream-watcher', `state save failed: ${e.message}`);
    }
  }
}

// Singleton — matches the pattern of health-checker.js so it integrates
// without a separate instantiation site.
module.exports = new CaddyUpstreamWatcher();
module.exports.CaddyUpstreamWatcher = CaddyUpstreamWatcher;