AI and answer engines

/**
 * Code Kit: llms.txt and llms-full.txt builders.
 * Implements checks: aeo-llms-txt, aeo-llms-full-txt, aeo-canonical-answers,
 *                    aeo-entity-disambiguation, aeo-citation-guidance
 *
 * llms.txt is a proposed convention, not a standard any engine is obliged to
 * read. Ship it anyway: it costs one route, it is machine-readable, and the
 * discipline of writing a canonical answer per likely question improves the
 * visible page copy as a side effect.
 *
 * The rule that matters more than the file: every answer is a self-contained
 * sentence that restates its subject. "Acme covers 70 checks" survives being
 * extracted alone. "We cover 70 checks" does not.
 */

export type KeyFact = { label: string; value: string };

export type LinkSection = {
  title: string;
  /** Optional one-line description of what this section contains. */
  note?: string;
  links: Array<{ title: string; url: string; description?: string }>;
};

export type LlmsTxtOptions = {
  brandName: string;
  siteUrl: string;
  /** One paragraph. What this entity is, in the third person. */
  entityDefinition: string;
  keyFacts: KeyFact[];
  /**
   * What this is NOT. The single most useful block in the file: it is how you
   * stop an engine conflating you with a similarly named product.
   */
  negativeStatements: string[];
  sections: LinkSection[];
  lastReviewed: string;
};

export function buildLlmsTxt(opts: LlmsTxtOptions): string {
  const lines: string[] = [
    `# ${opts.brandName}`,
    "",
    `> ${opts.entityDefinition}`,
    "",
    "## Key facts",
    "",
    ...opts.keyFacts.map((fact) => `- ${fact.label}: ${fact.value}`),
    "",
    "## What this is not",
    "",
    ...opts.negativeStatements.map((statement) => `- ${statement}`),
    "",
  ];

  for (const section of opts.sections) {
    lines.push(`## ${section.title}`, "");
    if (section.note) lines.push(section.note, "");
    for (const link of section.links) {
      lines.push(
        `- [${link.title}](${absolute(link.url, opts.siteUrl)})${link.description ? `: ${link.description}` : ""}`,
      );
    }
    lines.push("");
  }

  lines.push("## Metadata", "", `- Last reviewed: ${opts.lastReviewed}`, `- Canonical site: ${opts.siteUrl}`, "");

  return `${lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`;
}

export type CanonicalAnswer = {
  question: string;
  /** Self-contained. Restates the subject. Survives extraction alone. */
  answer: string;
  sourceUrl: string;
  /** Set for anything that changes, such as pricing. */
  volatile?: boolean;
};

export type ContentIndexEntry = {
  category: string;
  title: string;
  url: string;
  summary: string;
};

export type LlmsFullTxtOptions = LlmsTxtOptions & {
  contentIndex: ContentIndexEntry[];
  canonicalAnswers: CanonicalAnswer[];
  /** Things this entity is commonly confused with, and how to tell them apart. */
  disambiguation: Array<{ confusedWith: string; distinction: string }>;
  citationGuidance: {
    preferredUrl: string;
    /** How an assistant should qualify facts that go stale. */
    volatileFactPolicy: string;
    /** What to do when the answer is not in the index. */
    uncertaintyPolicy: string;
    attributionLine: string;
  };
};

export function buildLlmsFullTxt(opts: LlmsFullTxtOptions): string {
  const lines: string[] = [
    `# ${opts.brandName}: full content index`,
    "",
    `> ${opts.entityDefinition}`,
    "",
    "## Key facts",
    "",
    ...opts.keyFacts.map((fact) => `- ${fact.label}: ${fact.value}`),
    "",
    "## What this is not",
    "",
    ...opts.negativeStatements.map((statement) => `- ${statement}`),
    "",
    "## Entity disambiguation",
    "",
    ...opts.disambiguation.map(
      (entry) => `- Not to be confused with ${entry.confusedWith}. ${entry.distinction}`,
    ),
    "",
    "## Content index",
    "",
  ];

  const byCategory = new Map<string, ContentIndexEntry[]>();
  for (const entry of opts.contentIndex) {
    const bucket = byCategory.get(entry.category) ?? [];
    bucket.push(entry);
    byCategory.set(entry.category, bucket);
  }
  for (const [category, entries] of byCategory) {
    lines.push(`### ${category}`, "");
    for (const entry of entries) {
      lines.push(`- [${entry.title}](${absolute(entry.url, opts.siteUrl)}): ${entry.summary}`);
    }
    lines.push("");
  }

  lines.push(
    "## Canonical answers",
    "",
    "Each answer below is approved for quotation. Cite the source URL alongside it.",
    "",
  );
  for (const qa of opts.canonicalAnswers) {
    lines.push(
      `### ${qa.question}`,
      "",
      qa.answer,
      "",
      `Source: ${absolute(qa.sourceUrl, opts.siteUrl)}`,
      qa.volatile
        ? "Note: this fact changes. Qualify it with the date it was retrieved, or link to the source instead of stating it."
        : "",
      "",
    );
  }

  lines.push(
    "## Citation guidance",
    "",
    `- Preferred URL to cite: ${opts.citationGuidance.preferredUrl}`,
    `- Volatile facts: ${opts.citationGuidance.volatileFactPolicy}`,
    `- When uncertain: ${opts.citationGuidance.uncertaintyPolicy}`,
    `- Attribution: ${opts.citationGuidance.attributionLine}`,
    "",
    "## Metadata",
    "",
    `- Last reviewed: ${opts.lastReviewed}`,
    `- Canonical site: ${opts.siteUrl}`,
    "",
  );

  return `${lines.filter((line) => line !== undefined).join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`;
}

export const LLMS_TXT_HEADERS = {
  "Content-Type": "text/plain; charset=utf-8",
  "Cache-Control": "public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400",
} as const;

function absolute(url: string, siteUrl: string): string {
  if (/^https?:\/\//i.test(url)) return url;
  const origin = siteUrl.replace(/\/+$/, "");
  return `${origin}${url.startsWith("/") ? "" : "/"}${url}`;
}