dashcaddy.net/scripts/inject-structured-data.mjs

#!/usr/bin/env node
/**
 * inject-structured-data.mjs
 *
 * Adds JSON-LD structured data to every built DashCaddy app-guide page
 * (out/docs/catalog/<app>/index.html) after `next build` has produced the
 * static export.
 *
 * Design rules:
 *  - Runs on npm `postbuild`, so it operates on the real rendered HTML, not on
 *    the TSX source. Everything it emits is therefore provably derived from
 *    text a visitor can already see on the page (Google's requirement for
 *    FAQPage / HowTo markup).
 *  - Emits nothing it cannot ground. A page with no category badge gets no
 *    applicationCategory; a page with no troubleshooting list gets no FAQPage;
 *    a page with no dashboard install steps gets no HowTo. Missing fields are
 *    reported, never invented.
 *  - Idempotent: an already-injected page is detected and skipped.
 *
 * Never hand-edit this file's OUTPUT. Re-run the build.
 */

import { readFileSync, writeFileSync, readdirSync, statSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const HERE = dirname(fileURLToPath(import.meta.url));
const REPO = join(HERE, "..");
const OUT = join(REPO, "out");
const CATALOG = join(OUT, "docs", "catalog");

const ORIGIN = "https://dashcaddy.net";
const MARKER = 'data-dashcaddy-jsonld="app-guide"';
const BLOG_MARKER = 'data-dashcaddy-jsonld="blog-breadcrumb-faq"';

/* ------------------------------------------------------------------ text --- */

const ENTITIES = {
  amp: "&",
  lt: "<",
  gt: ">",
  quot: '"',
  apos: "'",
  nbsp: " ",
  "39": "'",
  "x27": "'",
  "x2F": "/",
  "47": "/",
};

function decodeEntities(s) {
  return s.replace(/&#?([a-zA-Z0-9]+);/g, (whole, code) => {
    if (Object.prototype.hasOwnProperty.call(ENTITIES, code)) return ENTITIES[code];
    if (Object.prototype.hasOwnProperty.call(ENTITIES, "#" + code)) {
      return ENTITIES["#" + code];
    }
    if (/^#?\d+$/.test(code)) {
      const n = Number(code.replace("#", ""));
      if (Number.isFinite(n) && n > 0 && n < 0x10ffff) return String.fromCodePoint(n);
    }
    if (/^#?x[0-9a-f]+$/i.test(code)) {
      const n = parseInt(code.replace(/^#?x/i, ""), 16);
      if (Number.isFinite(n) && n > 0 && n < 0x10ffff) return String.fromCodePoint(n);
    }
    return whole;
  });
}

/** Strip tags, decode entities, collapse whitespace. */
function text(html) {
  return decodeEntities(
    html
      .replace(/<br\s*\/?>/gi, " ")
      .replace(/<[^>]+>/g, " "),
  )
    .replace(/\s+/g, " ")
    .trim();
}

/** Tags stripped / entities decoded for the WHOLE document — used to prove that
 *  anything we mark up is genuinely visible to a visitor. */
function visibleText(doc) {
  return text(
    doc
      .replace(/<script\b[\s\S]*?<\/script>/gi, " ")
      .replace(/<style\b[\s\S]*?<\/style>/gi, " "),
  );
}

/* ------------------------------------------------------------- extraction --- */

/** Return the inner HTML of the first <tag ...> ... </tag> starting at `from`. */
function innerOf(html, tag, from = 0) {
  const open = new RegExp(`<${tag}\\b[^>]*>`, "gi");
  open.lastIndex = from;
  const m = open.exec(html);
  if (!m) return null;
  let depth = 1;
  const re = new RegExp(`<${tag}\\b[^>]*>|</${tag}>`, "gi");
  re.lastIndex = open.lastIndex;
  let mm;
  while ((mm = re.exec(html)) !== null) {
    if (mm[0].startsWith("</")) depth -= 1;
    else depth += 1;
    if (depth === 0) {
      return { inner: html.slice(open.lastIndex, mm.index), end: re.lastIndex };
    }
  }
  return null;
}

/** All top-level <li> entries inside an HTML fragment containing list items. */
function listItems(ulInner) {
  const items = [];
  let pos = 0;
  for (;;) {
    const li = innerOf(ulInner, "li", pos);
    if (!li) break;
    items.push(li.inner);
    pos = li.end;
  }
  return items;
}

/**
 * FAQ entries, taken ONLY from the visitor-visible troubleshooting list.
 *
 * The question is the page's own list label, verbatim (minus the trailing list
 * punctuation); the answer is the page's own explanation, verbatim. Nothing is
 * rephrased, and no question is synthesised — if the label is not visible on the
 * page, it does not get marked up. Pages whose troubleshooting entry has no bold
 * label are skipped.
 */
function extractFaqs(html) {
  const h = html.indexOf("<h2>Troubleshooting</h2>");
  if (h === -1) return [];
  const ul = innerOf(html, "ul", h);
  if (!ul) return [];
  const faqs = [];
  for (const li of listItems(ul.inner)) {
    // <strong>Label:</strong> answer
    const m = /<strong>([\s\S]*?)<\/strong>([\s\S]*)/.exec(li);
    if (!m) continue;
    const q = text(m[1]).replace(/[:\s]+$/, "");
    const a = text(m[2]);
    if (!q || !a) continue;
    faqs.push({ question: q, answer: a });
  }
  return faqs;
}

function extractSteps(html, appName) {
  const h = html.indexOf("<h2>Install via the DashCaddy dashboard</h2>");
  if (h === -1) return [];
  const ol = innerOf(html, "ol", h);
  if (!ol) return [];
  const steps = listItems(ol.inner)
    .map((li) => text(li))
    .filter((s) => s.length > 0);
  return steps.length >= 3 ? steps : [];
}

function extractMeta(html) {
  const title = /<title>([\s\S]*?)<\/title>/.exec(html);
  let appName = title ? text(title[1]) : "";
  appName = appName.replace(/\s*[—–-]\s*DashCaddy Docs\s*$/i, "").replace(/^Install\s+/i, "").trim();

  let description = "";
  const intro = /<p class="mt-4 max-w-3xl[^"]*">([\s\S]*?)<\/p>/.exec(html);
  if (intro) description = text(intro[1]);

  const cat = /Category:\s*([^<]+)</.exec(html);
  const diff = /Difficulty:\s*([^<]+)</.exec(html);
  const img = /Docker image:[\s\S]*?<code[^>]*>([\s\S]*?)<\/code>/.exec(html);

  return {
    appName,
    description,
    category: cat ? text(cat[1]) : null,
    difficulty: diff ? text(diff[1]) : null,
    dockerImage: img ? text(img[1]) : null,
  };
}

/* ------------------------------------------------------------ blog posts --- */

/**
 * Blog FAQ entries, taken only from a real, visible "Frequently asked questions"
 * section: each `### question` heading plus the paragraphs that follow it, both
 * verbatim. A post with no such section (or fewer than two entries) gets no
 * FAQPage node.
 */
function extractBlogFaqs(html) {
  const m = /<h2>(?:Frequently asked questions|FAQ)<\/h2>/i.exec(html);
  if (!m) return [];
  const rest = html.slice(m.index + m[0].length);
  let end = rest.length;
  for (const marker of ["<h2", "<footer", "<nav", "<aside", "<script", "<main"]) {
    const i = rest.indexOf(marker);
    if (i !== -1 && i < end) end = i;
  }
  const region = rest.slice(0, end);

  const faqs = [];
  const parts = region.split(/<h3>([\s\S]*?)<\/h3>/);
  for (let i = 1; i < parts.length; i += 2) {
    const q = text(parts[i]);
    const a = text(parts[i + 1] || "");
    if (q && a) faqs.push({ question: q, answer: a });
  }

  // Fallback shape: "**Question?** answer" rendered as <p><strong>…</strong>…</p>.
  if (faqs.length < 2) {
    const re = /<p><strong>([\s\S]*?)<\/strong>([\s\S]*?)<\/p>/g;
    let m2;
    while ((m2 = re.exec(region)) !== null) {
      const q = text(m2[1]);
      const a = text(m2[2]);
      if (q && q.endsWith("?") && a) faqs.push({ question: q, answer: a });
    }
  }

  return faqs.length >= 2 ? faqs : [];
}

function buildBlogGraph(slug, html, faqs) {
  const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/.exec(html);
  const title = h1 ? text(h1[1]) : slug;
  const page = `${ORIGIN}/blog/${slug}/`;

  const graph = [
    {
      "@type": "BreadcrumbList",
      "@id": `${page}#breadcrumb`,
      itemListElement: [
        { "@type": "ListItem", position: 1, name: "DashCaddy", item: `${ORIGIN}/` },
        { "@type": "ListItem", position: 2, name: "Blog", item: `${ORIGIN}/blog/` },
        { "@type": "ListItem", position: 3, name: title, item: page },
      ],
    },
  ];

  if (faqs.length >= 2) {
    graph.push({
      "@type": "FAQPage",
      "@id": `${page}#faq`,
      mainEntity: faqs.map((f) => ({
        "@type": "Question",
        name: f.question,
        acceptedAnswer: { "@type": "Answer", text: f.answer },
      })),
    });
  }

  return { "@context": "https://schema.org", "@graph": graph };
}

/* ------------------------------------------------------------------ build --- */

function buildGraph(appId, meta, faqs, steps) {
  const page = `${ORIGIN}/docs/catalog/${appId}/`;
  const graph = [];

  graph.push({
    "@type": "BreadcrumbList",
    "@id": `${page}#breadcrumb`,
    itemListElement: [
      { "@type": "ListItem", position: 1, name: "DashCaddy", item: `${ORIGIN}/` },
      { "@type": "ListItem", position: 2, name: "Documentation", item: `${ORIGIN}/docs/` },
      { "@type": "ListItem", position: 3, name: "App Catalog", item: `${ORIGIN}/docs/catalog/` },
      { "@type": "ListItem", position: 4, name: `Install ${meta.appName}`, item: page },
    ],
  });

  const app = {
    "@type": "SoftwareApplication",
    "@id": `${page}#software`,
    name: meta.appName,
    url: page,
    operatingSystem: "Linux",
    applicationCategory: meta.category || "UtilitiesApplication",
  };
  if (meta.description) app.description = meta.description;
  if (meta.dockerImage) app.softwareRequirements = `Docker image ${meta.dockerImage}`;
  graph.push(app);

  if (faqs.length >= 2) {
    graph.push({
      "@type": "FAQPage",
      "@id": `${page}#faq`,
      mainEntity: faqs.map((f) => ({
        "@type": "Question",
        name: f.question,
        acceptedAnswer: { "@type": "Answer", text: f.answer },
      })),
    });
  }

  if (steps.length >= 3) {
    graph.push({
      "@type": "HowTo",
      "@id": `${page}#howto`,
      name: `Install ${meta.appName} with DashCaddy`,
      description: `Deploy ${meta.appName} on a DashCaddy host from the dashboard.`,
      step: steps.map((s, i) => ({
        "@type": "HowToStep",
        position: i + 1,
        name: `Step ${i + 1}`,
        text: s,
      })),
    });
  }

  return { "@context": "https://schema.org", "@graph": graph };
}

/* ------------------------------------------------------------------- main --- */

function main() {
  if (!statSync(OUT, { throwIfNoEntry: false })) {
    console.error("inject-structured-data: out/ not found — run next build first");
    process.exit(1);
  }

  const appIds = readdirSync(CATALOG).filter((d) => {
    try {
      return statSync(join(CATALOG, d)).isDirectory();
    } catch {
      return false;
    }
  });

  if (appIds.length < 50) {
    console.error(`inject-structured-data: only ${appIds.length} app dirs found; refusing`);
    process.exit(1);
  }

  const report = { pages: 0, injected: 0, skipped: 0, faq: 0, howto: 0, image: 0, missing: [], ungrounded: 0 };

  for (const appId of appIds) {
    const file = join(CATALOG, appId, "index.html");
    if (!statSync(file, { throwIfNoEntry: false })) {
      report.missing.push(`${appId}: no index.html`);
      continue;
    }
    report.pages += 1;
    let html = readFileSync(file, "utf8");

    if (html.includes(MARKER)) {
      report.skipped += 1;
      continue;
    }

    const meta = extractMeta(html);
    const vis = visibleText(html);
    const faqs = extractFaqs(html).filter((f) => {
      if (vis.includes(f.question) && vis.includes(f.answer)) return true;
      report.ungrounded += 1;
      return false;
    });
    const steps = extractSteps(html, meta.appName).filter((s) => {
      if (vis.includes(s)) return true;
      report.ungrounded += 1;
      return false;
    });

    if (!meta.appName) {
      report.missing.push(`${appId}: no app name`);
      continue;
    }
    const gaps = [];
    if (!meta.category) gaps.push("category");
    if (!meta.difficulty) gaps.push("difficulty");
    if (!meta.dockerImage) gaps.push("dockerImage");
    if (gaps.length) report.missing.push(`${appId}: ${gaps.join(",")}`);
    if (faqs.length >= 2) report.faq += 1;
    if (steps.length >= 3) report.howto += 1;
    if (meta.dockerImage) report.image += 1;

    const json = JSON.stringify(buildGraph(appId, meta, faqs, steps))
      .replace(/</g, "\\u003c")
      .replace(/>/g, "\\u003e")
      .replace(/&/g, "\\u0026");

    const block =
      `<script type="application/ld+json" ${MARKER}>${json}</script>\n`;

    if (!html.includes("</head>")) {
      report.missing.push(`${appId}: no </head>`);
      continue;
    }
    html = html.replace("</head>", `${block}</head>`);
    writeFileSync(file, html);
    report.injected += 1;
  }

  console.log("inject-structured-data: app guide pages");
  console.log(`  pages_seen=${report.pages}`);
  console.log(`  pages_injected=${report.injected}`);
  console.log(`  pages_skipped_already=${report.skipped}`);
  console.log(`  pages_with_faqpage=${report.faq}`);
  console.log(`  pages_with_howto=${report.howto}`);
  console.log(`  pages_with_docker_image=${report.image}`);
  console.log(`  pages_with_gaps=${report.missing.length}`);
  console.log(`  ungrounded_strings_dropped=${report.ungrounded}`);
  for (const m of report.missing) console.log(`    gap: ${m}`);

  if (report.injected + report.skipped !== report.pages) {
    console.error("inject-structured-data: not every page was injected or skipped");
    process.exit(1);
  }

  injectBlogPosts();
}

/* ------------------------------------------------------------- blog pass --- */

function injectBlogPosts() {
  const blogDir = join(OUT, "blog");
  if (!statSync(blogDir, { throwIfNoEntry: false })) {
    console.log("inject-structured-data: no out/blog dir — skipping blog pass");
    return;
  }

  const slugs = readdirSync(blogDir).filter((d) => {
    try {
      return statSync(join(blogDir, d)).isDirectory();
    } catch {
      return false;
    }
  });

  let seen = 0;
  let injected = 0;
  let skipped = 0;
  let withFaq = 0;
  let questions = 0;
  let ungrounded = 0;
  const gaps = [];

  for (const slug of slugs) {
    const file = join(blogDir, slug, "index.html");
    if (!statSync(file, { throwIfNoEntry: false })) {
      gaps.push(`${slug}: no index.html`);
      continue;
    }
    seen += 1;
    let html = readFileSync(file, "utf8");
    if (html.includes(BLOG_MARKER)) {
      skipped += 1;
      continue;
    }

    const vis = visibleText(html);
    const faqs = extractBlogFaqs(html).filter((f) => {
      if (vis.includes(f.question) && vis.includes(f.answer)) return true;
      ungrounded += 1;
      return false;
    });
    if (faqs.length >= 2) {
      withFaq += 1;
      questions += faqs.length;
    } else {
      gaps.push(`${slug}: no visible FAQ section`);
    }

    const json = JSON.stringify(buildBlogGraph(slug, html, faqs))
      .replace(/</g, "\\u003c")
      .replace(/>/g, "\\u003e")
      .replace(/&/g, "\\u0026");
    const block = `<script type="application/ld+json" ${BLOG_MARKER}>${json}</script>\n`;

    if (!html.includes("</head>")) {
      gaps.push(`${slug}: no </head>`);
      continue;
    }
    writeFileSync(file, html.replace("</head>", `${block}</head>`));
    injected += 1;
  }

  console.log("inject-structured-data: blog posts");
  console.log(`  posts_seen=${seen}`);
  console.log(`  posts_injected=${injected}`);
  console.log(`  posts_skipped_already=${skipped}`);
  console.log(`  posts_with_faqpage=${withFaq}`);
  console.log(`  faq_questions=${questions}`);
  console.log(`  ungrounded_strings_dropped=${ungrounded}`);
  console.log(`  notes=${gaps.length}`);
  for (const g of gaps) console.log(`    note: ${g}`);
}

main();