The SEO playbookfor Solo SaaS founders

/**
 * Code Kit: the sitewide @graph composer.
 * Implements checks: schema-graph-anchors, schema-single-script, schema-no-duplicates
 *
 * One <script type="application/ld+json"> per page holding one @graph array.
 * Nodes cross-reference each other by @id instead of duplicating themselves,
 * which is what stops a site from shipping the same Organization object 2,000
 * times and lets a crawler resolve one entity across every page.
 */

import { compact } from "../constants";
import { organization, website, type OrganizationOptions, type WebSiteOptions } from "./organization";
import { product, type ProductOptions } from "./commerce";
import { type JsonLdObject, SCHEMA_CONTEXT, nodeId, ref } from "./types";

export type SiteGraphOptions = {
  siteUrl: string;
  organization: Omit<OrganizationOptions, "siteUrl">;
  website: Omit<WebSiteOptions, "siteUrl" | "publisher">;
  /** Omit on sites that do not sell a product. */
  product?: Omit<ProductOptions, "siteUrl">;
  /** Page-level nodes: WebPage, BreadcrumbList, Article, FAQPage, and so on. */
  additionalNodes?: Array<JsonLdObject | undefined>;
};

/**
 * Compose the sitewide graph.
 *
 * The three anchors are stable and cross-referenced:
 *   #organization  is the publisher of #website and the brand of #product
 *   #website       is the isPartOf target for every page node
 *   #product       is the thing being sold
 */
export function siteGraph(opts: SiteGraphOptions): JsonLdObject {
  const orgId = nodeId(opts.siteUrl, "#organization");

  const nodes: Array<JsonLdObject | undefined> = [
    organization({ siteUrl: opts.siteUrl, ...opts.organization }),
    website({ siteUrl: opts.siteUrl, publisher: ref(orgId), ...opts.website }),
    opts.product
      ? product({
          siteUrl: opts.siteUrl,
          ...opts.product,
          brandName: opts.product.brandName,
        })
      : undefined,
    ...(opts.additionalNodes ?? []),
  ];

  return {
    "@context": SCHEMA_CONTEXT,
    "@graph": dedupeById(nodes.filter(isPresent)) as unknown as JsonLdObject[],
  } as unknown as JsonLdObject;
}

/**
 * Build a page-level graph that references the sitewide anchors without
 * repeating them. Use on every page that is not the home page.
 */
export function pageGraph(opts: {
  siteUrl: string;
  nodes: Array<JsonLdObject | undefined>;
}): JsonLdObject {
  return {
    "@context": SCHEMA_CONTEXT,
    "@graph": dedupeById(opts.nodes.filter(isPresent)) as unknown as JsonLdObject[],
  } as unknown as JsonLdObject;
}

function isPresent<T>(value: T | undefined | null): value is T {
  return value !== undefined && value !== null;
}

/** Two nodes with the same @id in one graph is a validation error. Last write wins. */
function dedupeById(nodes: JsonLdObject[]): JsonLdObject[] {
  const seen = new Map<string, JsonLdObject>();
  const anonymous: JsonLdObject[] = [];
  for (const node of nodes) {
    const id = typeof node["@id"] === "string" ? node["@id"] : undefined;
    if (id) seen.set(id, node);
    else anonymous.push(node);
  }
  return [...seen.values(), ...anonymous];
}

/**
 * Collect every @id declared in a graph and every @id referenced by it, so a
 * test can assert that no reference dangles.
 *
 * The distinction that matters: an object carrying `@id` plus other properties
 * DECLARES a node, at any nesting depth. An object carrying nothing but `@id`
 * REFERENCES one. A nested ImageObject with its own `@id` is a declaration, and
 * treating it as a dangling reference would be a false positive.
 */
export function graphIdIntegrity(graph: JsonLdObject): {
  declared: Set<string>;
  referenced: Set<string>;
  dangling: string[];
} {
  const declared = new Set<string>();
  const referenced = new Set<string>();
  const nodes = (graph["@graph"] as unknown as JsonLdObject[]) ?? [];

  const walk = (value: unknown): void => {
    if (Array.isArray(value)) {
      for (const item of value) walk(item);
      return;
    }
    if (!value || typeof value !== "object") return;

    const obj = value as Record<string, unknown>;
    const keys = Object.keys(obj);
    const id = typeof obj["@id"] === "string" ? obj["@id"] : undefined;

    if (id) {
      if (keys.length === 1) referenced.add(id);
      else declared.add(id);
    }

    for (const [key, val] of Object.entries(obj)) {
      if (key === "@id") continue;
      walk(val);
    }
  };

  for (const node of nodes) walk(node);

  return {
    declared,
    referenced,
    dangling: [...referenced].filter((id) => !declared.has(id)),
  };
}

export { compact };

/**
 * Code Kit: bot-economics robots policy.
 * Implements checks: crawl-robots-generated, crawl-bot-economics,
 *                    crawl-private-routes-disallowed, crawl-sitemap-declared
 *
 * The decision nobody makes deliberately: which crawlers get your content.
 * The useful split is not "AI bots" versus "search bots". It is whether a
 * crawler can send you traffic or a citation in return for the bandwidth.
 *
 *   Search crawlers      index you and send clicks.            Allow.
 *   Citation crawlers    fetch a page to answer a live query
 *                        and name the source.                  Allow.
 *   Training crawlers    build a model. No link, no click,
 *                        no attribution.                       Your call.
 *
 * Blocking a training crawler does not remove you from that company's answer
 * engine if its citation crawler is a separate user agent, which for OpenAI,
 * Anthropic, Perplexity and Google it currently is. That asymmetry is the whole
 * mechanism.
 */

export type CrawlerClass = "search" | "citation" | "training" | "seo-tool" | "archive";

export type CrawlerProfile = {
  userAgent: string;
  operator: string;
  class: CrawlerClass;
  /** Does allowing this crawler produce visits or citations? */
  sendsReferralTraffic: boolean;
  purpose: string;
  documentationUrl?: string;
};

/**
 * Crawler registry.
 *
 * Verified against each operator's published documentation. Recheck quarterly:
 * user agent names change, and new ones appear faster than anyone updates
 * robots.txt.
 */
export const CRAWLERS: CrawlerProfile[] = [
  {
    userAgent: "Googlebot",
    operator: "Google",
    class: "search",
    sendsReferralTraffic: true,
    purpose: "Google Search index. Also feeds AI Overviews.",
    documentationUrl: "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
  },
  {
    userAgent: "Bingbot",
    operator: "Microsoft",
    class: "search",
    sendsReferralTraffic: true,
    purpose: "Bing index. Also the retrieval layer behind Copilot.",
    documentationUrl: "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
  },
  {
    userAgent: "DuckDuckBot",
    operator: "DuckDuckGo",
    class: "search",
    sendsReferralTraffic: true,
    purpose: "DuckDuckGo index.",
  },
  {
    userAgent: "Applebot",
    operator: "Apple",
    class: "search",
    sendsReferralTraffic: true,
    purpose: "Siri and Spotlight suggestions.",
    documentationUrl: "https://support.apple.com/en-us/119829",
  },
  {
    userAgent: "OAI-SearchBot",
    operator: "OpenAI",
    class: "citation",
    sendsReferralTraffic: true,
    purpose: "ChatGPT search index. Links back to the source.",
    documentationUrl: "https://platform.openai.com/docs/bots",
  },
  {
    userAgent: "ChatGPT-User",
    operator: "OpenAI",
    class: "citation",
    sendsReferralTraffic: true,
    purpose: "Live fetch when a user asks ChatGPT about a specific URL.",
    documentationUrl: "https://platform.openai.com/docs/bots",
  },
  {
    userAgent: "PerplexityBot",
    operator: "Perplexity",
    class: "citation",
    sendsReferralTraffic: true,
    purpose: "Perplexity answer index. Cites sources inline.",
    documentationUrl: "https://docs.perplexity.ai/guides/bots",
  },
  {
    userAgent: "ClaudeBot",
    operator: "Anthropic",
    class: "citation",
    sendsReferralTraffic: true,
    purpose: "Claude retrieval and citation.",
    documentationUrl: "https://support.anthropic.com/en/articles/8896518",
  },
  {
    userAgent: "Claude-User",
    operator: "Anthropic",
    class: "citation",
    sendsReferralTraffic: true,
    purpose: "Live fetch when a Claude user asks about a specific URL.",
  },
  {
    userAgent: "Applebot-Extended",
    operator: "Apple",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Apple foundation model training. Opting out does not affect Siri.",
    documentationUrl: "https://support.apple.com/en-us/119829",
  },
  {
    userAgent: "GPTBot",
    operator: "OpenAI",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Model training corpus. Separate from OAI-SearchBot.",
    documentationUrl: "https://platform.openai.com/docs/bots",
  },
  {
    userAgent: "Google-Extended",
    operator: "Google",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Gemini training. Blocking it does not remove you from Search.",
    documentationUrl: "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
  },
  {
    userAgent: "anthropic-ai",
    operator: "Anthropic",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Legacy training crawler identifier.",
  },
  {
    userAgent: "CCBot",
    operator: "Common Crawl",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Open crawl corpus used as training data by many labs.",
    documentationUrl: "https://commoncrawl.org/ccbot",
  },
  {
    userAgent: "Bytespider",
    operator: "ByteDance",
    class: "training",
    sendsReferralTraffic: false,
    purpose: "Training corpus. Widely reported to ignore robots.txt.",
  },
  {
    userAgent: "AhrefsBot",
    operator: "Ahrefs",
    class: "seo-tool",
    sendsReferralTraffic: false,
    purpose: "Backlink index. Allowing it makes your own backlink data visible to you and to competitors.",
  },
  {
    userAgent: "SemrushBot",
    operator: "Semrush",
    class: "seo-tool",
    sendsReferralTraffic: false,
    purpose: "Competitive index. Same trade as AhrefsBot.",
  },
  {
    userAgent: "ia_archiver",
    operator: "Internet Archive",
    class: "archive",
    sendsReferralTraffic: false,
    purpose: "Wayback Machine preservation.",
  },
];

export type RobotsPolicy = {
  /** Crawler classes permitted to crawl. */
  allowClasses: CrawlerClass[];
  /** Paths every permitted crawler is denied. */
  disallowPaths: string[];
  /** Absolute sitemap URLs. */
  sitemaps: string[];
  /** Canonical host, no scheme. Bing reads this; Google ignores it. */
  host?: string;
  /** Extra user agents to deny outright, beyond the class rules. */
  denyUserAgents?: string[];
  /** Emit the reasoning as comments. Leave on: people read this file. */
  annotate?: boolean;
  crawlers?: CrawlerProfile[];
};

export const RECOMMENDED_POLICY: Pick<RobotsPolicy, "allowClasses"> = {
  // Allow anything that can send a visit or a citation. Deny training-only
  // crawlers and commercial SEO indexes, which take bandwidth and return nothing.
  allowClasses: ["search", "citation"],
};

export type RobotsRule = {
  userAgent: string | string[];
  allow?: string | string[];
  disallow?: string | string[];
};

/** Structured output, for `app/robots.ts` in Next.js. */
export function robotsRules(policy: RobotsPolicy): RobotsRule[] {
  const crawlers = policy.crawlers ?? CRAWLERS;
  const allowed = crawlers.filter((c) => policy.allowClasses.includes(c.class));
  const denied = crawlers.filter((c) => !policy.allowClasses.includes(c.class));
  const denyList = [
    ...denied.map((c) => c.userAgent),
    ...(policy.denyUserAgents ?? []),
  ];

  const rules: RobotsRule[] = [
    // The wildcard rule governs every crawler not named below, including ones
    // that do not exist yet.
    { userAgent: "*", allow: "/", disallow: policy.disallowPaths },
  ];

  for (const crawler of allowed) {
    rules.push({ userAgent: crawler.userAgent, allow: "/", disallow: policy.disallowPaths });
  }
  for (const userAgent of denyList) {
    rules.push({ userAgent, disallow: "/" });
  }

  return rules;
}

/** Plain text output, for a static robots.txt or the generator tool. */
export function robotsTxt(policy: RobotsPolicy): string {
  const crawlers = policy.crawlers ?? CRAWLERS;
  const annotate = policy.annotate ?? true;
  const lines: string[] = [];

  if (annotate) {
    lines.push(
      "# robots.txt",
      "#",
      "# Policy: allow any crawler that can return a visit or a citation.",
      "# Deny crawlers that consume bandwidth and return neither.",
      "#",
    );
  }

  const emit = (rule: RobotsRule, comment?: string) => {
    if (annotate && comment) lines.push(`# ${comment}`);
    const agents = Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent];
    for (const agent of agents) lines.push(`User-agent: ${agent}`);
    for (const allow of toArray(rule.allow)) lines.push(`Allow: ${allow}`);
    for (const disallow of toArray(rule.disallow)) lines.push(`Disallow: ${disallow}`);
    lines.push("");
  };

  emit(
    { userAgent: "*", allow: "/", disallow: policy.disallowPaths },
    "Default policy for every crawler not named below.",
  );

  for (const crawler of crawlers.filter((c) => policy.allowClasses.includes(c.class))) {
    emit(
      { userAgent: crawler.userAgent, allow: "/", disallow: policy.disallowPaths },
      `${crawler.operator}: ${crawler.purpose}`,
    );
  }

  const denied = crawlers.filter((c) => !policy.allowClasses.includes(c.class));
  for (const crawler of denied) {
    emit(
      { userAgent: crawler.userAgent, disallow: "/" },
      `${crawler.operator}: ${crawler.purpose} No referral traffic.`,
    );
  }
  for (const userAgent of policy.denyUserAgents ?? []) {
    emit({ userAgent, disallow: "/" });
  }

  for (const sitemap of policy.sitemaps) lines.push(`Sitemap: ${sitemap}`);
  if (policy.host) lines.push(`Host: ${policy.host}`);

  return `${lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`;
}

function toArray(value: string | string[] | undefined): string[] {
  if (!value) return [];
  return Array.isArray(value) ? value : [value];
}