Hugo Agent Readiness Playbook: From Score 8 to 83

# YOUR SITE NAME

> One sentence: what the site is and who it's for.

## About

- Key belief or value
- Another belief or value

## Site structure

### Core pages

- [Home](https://YOUR-DOMAIN/) — description
- [About](https://YOUR-DOMAIN/about/) — description
- [Section](https://YOUR-DOMAIN/section/) — description

### Subsections

- [Section › Sub](https://YOUR-DOMAIN/section/sub/) — description

## Machine-readable resources

- [Sitemap](https://YOUR-DOMAIN/sitemap.xml) — Full XML sitemap
- [RSS Feed](https://YOUR-DOMAIN/feed.xml) — Recent updates
- [Search Index](https://YOUR-DOMAIN/search.json) — Full-text search index (JSON)
- [API Catalog](https://YOUR-DOMAIN/.well-known/api-catalog) — RFC 9727 API catalog
- [Agent Skills](https://YOUR-DOMAIN/.well-known/agent-skills/index.json) — Agent capabilities

## Usage guidelines for LLMs

- Content is free to read, reference, and cite with attribution.
- Training on content requires attribution; commercial training requires explicit permission.
- Skip navigation, sidebars, and footer elements.

## Contact

- Email: [you@example.com](mailto:you@example.com)

{
  "linkset": [
    {
      "anchor": "https://YOUR-DOMAIN",
      "service-desc": [
        {
          "href": "https://YOUR-DOMAIN/search.json",
          "type": "application/json",
          "title": "Full-text search index"
        }
      ],
      "service-doc": [
        {
          "href": "https://YOUR-DOMAIN/llms.txt",
          "type": "text/markdown",
          "title": "LLM content index"
        }
      ],
      "alternate": [
        {
          "href": "https://YOUR-DOMAIN/feed.xml",
          "type": "application/rss+xml",
          "title": "RSS feed"
        },
        {
          "href": "https://YOUR-DOMAIN/sitemap.xml",
          "type": "application/xml",
          "title": "Sitemap"
        }
      ]
    }
  ]
}

{
  "serverInfo": {
    "name": "YOUR SITE NAME",
    "version": "1.0.0"
  },
  "description": "One paragraph describing what the site is.",
  "url": "https://YOUR-DOMAIN",
  "transport": {
    "type": "http",
    "endpoint": "https://YOUR-DOMAIN/.well-known/agent-skills/index.json"
  },
  "capabilities": {
    "resources": true,
    "tools": false,
    "prompts": false
  },
  "resources": [
    {
      "name": "Search Index",
      "uri": "https://YOUR-DOMAIN/search.json",
      "description": "Full-text search index of all site content (JSON)",
      "mimeType": "application/json"
    },
    {
      "name": "LLM Index",
      "uri": "https://YOUR-DOMAIN/llms.txt",
      "description": "Structured Markdown overview for LLM ingestion",
      "mimeType": "text/markdown"
    }
  ],
  "content_policy": {
    "ai_inference": "allow",
    "ai_search": "allow",
    "ai_training": "disallow",
    "attribution_required": true
  }
}

{
  "schema_version": "1.0",
  "site": "https://YOUR-DOMAIN",
  "skills": [
    {
      "id": "search-content",
      "name": "Search Site Content",
      "description": "Search all content using a full-text index.",
      "endpoint": "https://YOUR-DOMAIN/search.json",
      "method": "GET",
      "response_type": "application/json",
      "usage": "Fetch the JSON array. Each entry has: uri, title, content (up to 1000 chars), description, tags. Filter client-side."
    },
    {
      "id": "browse-feed",
      "name": "Browse Recent Updates",
      "description": "Get the latest content updates in RSS format.",
      "endpoint": "https://YOUR-DOMAIN/feed.xml",
      "method": "GET",
      "response_type": "application/rss+xml",
      "usage": "Standard RSS 2.0. Parse <item> elements for title, link, description, pubDate."
    },
    {
      "id": "list-pages",
      "name": "List All Pages",
      "description": "Full list of all pages with last-modified dates.",
      "endpoint": "https://YOUR-DOMAIN/sitemap.xml",
      "method": "GET",
      "response_type": "application/xml",
      "usage": "XML sitemap. Each <url> has <loc> and <lastmod>."
    },
    {
      "id": "read-llms-index",
      "name": "Read LLM Content Index",
      "description": "Structured Markdown overview of the site for LLM comprehension.",
      "endpoint": "https://YOUR-DOMAIN/llms.txt",
      "method": "GET",
      "response_type": "text/markdown",
      "usage": "Fetch and parse as Markdown. Contains site description, section links, and usage guidelines."
    }
  ]
}

<!-- Agent Discoverability -->
<link rel="alternate" type="text/markdown" title="LLM-friendly content index" href="{{ "llms.txt" | absURL }}" />
<link rel="sitemap" type="application/xml" title="Sitemap" href="{{ "sitemap.xml" | absURL }}" />
<link rel="api-catalog" href="{{ "/.well-known/api-catalog" | absURL }}" />
<meta name="robots" content="index, follow" />
<meta name="ai-inference" content="allow" />
<meta name="ai-search" content="allow" />
<meta name="ai-training" content="disallow" />

<!-- WebMCP: expose site tools to AI agents -->
<script>
  (function () {
    if (!("modelContext" in navigator)) return;

    navigator.modelContext.provideContext({
      tools: [
        {
          name: "search_SITENAME",
          description:
            "Search site content. Returns matching pages with title, URL, and snippet.",
          inputSchema: {
            type: "object",
            properties: {
              query: { type: "string", description: "Search term" },
            },
            required: ["query"],
          },
          execute: async function ({ query }) {
            const res = await fetch("/search.json");
            const data = await res.json();
            const q = query.toLowerCase();
            return data
              .filter(function (item) {
                return (
                  (item.title && item.title.toLowerCase().includes(q)) ||
                  (item.content && item.content.toLowerCase().includes(q)) ||
                  (item.description &&
                    item.description.toLowerCase().includes(q)) ||
                  (item.tags &&
                    item.tags.some(function (t) {
                      return t.toLowerCase().includes(q);
                    }))
                );
              })
              .slice(0, 10)
              .map(function (item) {
                return {
                  title: item.title,
                  url: "https://YOUR-DOMAIN" + item.uri,
                  description:
                    item.description ||
                    (item.content ? item.content.slice(0, 200) + "…" : ""),
                  tags: item.tags || [],
                };
              });
          },
        },
        {
          name: "get_recent_updates",
          description: "Get recently updated pages, sorted by date.",
          inputSchema: {
            type: "object",
            properties: {
              limit: {
                type: "number",
                description: "Max results (default 10)",
              },
            },
          },
          execute: async function ({ limit }) {
            const res = await fetch("/feed.xml");
            const text = await res.text();
            const parser = new DOMParser();
            const doc = parser.parseFromString(text, "application/xml");
            return Array.from(doc.querySelectorAll("item"))
              .slice(0, limit || 10)
              .map(function (item) {
                return {
                  title: item.querySelector("title")?.textContent || "",
                  url: item.querySelector("link")?.textContent || "",
                  date: item.querySelector("pubDate")?.textContent || "",
                };
              });
          },
        },
      ],
    });
  })();
</script>

/**
 * Cloudflare Pages Middleware — Markdown Content Negotiation
 *
 * When a request includes `Accept: text/markdown`, fetches the HTML page,
 * strips chrome (nav, header, footer, scripts, styles, cookie banners),
 * converts the main content to clean Markdown, and returns it with
 * Content-Type: text/markdown and x-markdown-tokens headers.
 *
 * Browsers receive normal HTML (no Accept: text/markdown header sent).
 *
 * Spec: https://llmstxt.org/
 * Docs: https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/
 */

export async function onRequest(context) {
  const { request, next } = context;

  const accept = request.headers.get("Accept") || "";
  if (!accept.includes("text/markdown")) {
    return next();
  }

  const method = request.method;
  if (method !== "GET" && method !== "HEAD") {
    return next();
  }

  const url = new URL(request.url);
  const path = url.pathname;

  // Skip assets
  if (
    /\.(css|js|png|jpg|jpeg|gif|svg|webp|ico|woff2?|ttf|eot|json|xml|txt|pdf)$/i.test(
      path,
    )
  ) {
    return next();
  }

  const response = await next();

  const contentType = response.headers.get("Content-Type") || "";
  if (!contentType.includes("text/html")) {
    return response;
  }

  const html = await response.text();
  const markdown = htmlToMarkdown(html, url.href);
  const tokenEstimate = Math.ceil(markdown.length / 4);

  return new Response(markdown, {
    status: response.status,
    headers: {
      "Content-Type": "text/markdown; charset=utf-8",
      "x-markdown-tokens": String(tokenEstimate),
      "Cache-Control": "no-store",
      Vary: "Accept",
    },
  });
}

function htmlToMarkdown(html, pageUrl) {
  html = stripElements(html, [
    "script",
    "style",
    "noscript",
    "nav",
    "header",
    "footer",
    "aside",
    /<div[^>]*(?:cookie|consent|gdpr|banner|notice|overlay|modal)[^>]*>[\s\S]*?<\/div>/gi,
  ]);

  const main =
    extractTag(html, "main") ||
    extractTag(html, "article") ||
    extractTag(html, "body") ||
    html;

  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
  const pageTitle = titleMatch ? decodeEntities(titleMatch[1].trim()) : "";

  let md = convertToMarkdown(main);

  const header = [
    `<!-- Source: ${pageUrl} -->`,
    pageTitle ? `# ${pageTitle}\n` : "",
  ]
    .filter(Boolean)
    .join("\n");

  md = header + "\n\n" + md;
  md = md.replace(/\n{3,}/g, "\n\n").trim();

  return md;
}

function stripElements(html, targets) {
  for (const target of targets) {
    if (typeof target === "string") {
      const re = new RegExp(
        `<${target}(\\s[^>]*)?>([\\s\\S]*?)<\\/${target}>`,
        "gi",
      );
      html = html.replace(re, "");
    } else if (target instanceof RegExp) {
      html = html.replace(target, "");
    }
  }
  return html;
}

function extractTag(html, tag) {
  const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, "i");
  const m = html.match(re);
  return m ? m[1] : null;
}

function convertToMarkdown(html) {
  let md = html;

  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, (_, t) => `\n# ${clean(t)}\n`);
  md = md.replace(
    /<h2[^>]*>([\s\S]*?)<\/h2>/gi,
    (_, t) => `\n## ${clean(t)}\n`,
  );
  md = md.replace(
    /<h3[^>]*>([\s\S]*?)<\/h3>/gi,
    (_, t) => `\n### ${clean(t)}\n`,
  );
  md = md.replace(
    /<h4[^>]*>([\s\S]*?)<\/h4>/gi,
    (_, t) => `\n#### ${clean(t)}\n`,
  );
  md = md.replace(
    /<h5[^>]*>([\s\S]*?)<\/h5>/gi,
    (_, t) => `\n##### ${clean(t)}\n`,
  );
  md = md.replace(
    /<h6[^>]*>([\s\S]*?)<\/h6>/gi,
    (_, t) => `\n###### ${clean(t)}\n`,
  );

  md = md.replace(
    /<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi,
    (_, t) =>
      clean(t)
        .split("\n")
        .map((l) => `> ${l}`)
        .join("\n") + "\n",
  );

  md = md.replace(
    /<pre[^>]*><code(?:\s[^>]*)?>([\s\S]*?)<\/code><\/pre>/gi,
    (_, t) => `\n\`\`\`\n${decodeEntities(t)}\n\`\`\`\n`,
  );
  md = md.replace(
    /<code[^>]*>([\s\S]*?)<\/code>/gi,
    (_, t) => `\`${decodeEntities(t)}\``,
  );

  md = md.replace(
    /<(strong|b)[^>]*>([\s\S]*?)<\/(strong|b)>/gi,
    (_, _t, t) => `**${clean(t)}**`,
  );
  md = md.replace(
    /<(em|i)[^>]*>([\s\S]*?)<\/(em|i)>/gi,
    (_, _t, t) => `_${clean(t)}_`,
  );

  md = md.replace(
    /<a\s[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi,
    (_, href, text) => {
      const t = clean(text);
      return t ? `[${t}](${href})` : "";
    },
  );

  md = md.replace(/<img\s[^>]*alt="([^"]*)"[^>]*\/?>/gi, (_, alt) =>
    alt ? `_[Image: ${alt}]_` : "",
  );

  md = md.replace(/<hr[^>]*\/?>/gi, "\n---\n");
  md = md.replace(/<table[^>]*>([\s\S]*?)<\/table>/gi, convertTable);
  md = md.replace(
    /<ul[^>]*>([\s\S]*?)<\/ul>/gi,
    (_, inner) => convertList(inner, false) + "\n",
  );
  md = md.replace(
    /<ol[^>]*>([\s\S]*?)<\/ol>/gi,
    (_, inner) => convertList(inner, true) + "\n",
  );
  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, t) => `- ${clean(t)}\n`);

  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, t) => {
    const text = clean(t);
    return text ? `\n${text}\n` : "";
  });
  md = md.replace(/<div[^>]*>([\s\S]*?)<\/div>/gi, (_, t) => {
    const text = clean(t);
    return text ? `\n${text}\n` : "";
  });

  md = md.replace(/<br\s*\/?>/gi, "\n");
  md = md.replace(/<[^>]+>/g, "");
  md = decodeEntities(md);

  return md;
}

function convertList(inner, ordered) {
  const items = [];
  const re = /<li[^>]*>([\s\S]*?)<\/li>/gi;
  let i = 1,
    m;
  while ((m = re.exec(inner)) !== null) {
    const text = clean(m[1]);
    if (text) {
      items.push(ordered ? `${i}. ${text}` : `- ${text}`);
      i++;
    }
  }
  return items.join("\n");
}

function convertTable(_, inner) {
  const rows = [];
  const rowRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
  let rm;
  while ((rm = rowRe.exec(inner)) !== null) {
    const cells = [];
    const cellRe = /<t[hd][^>]*>([\s\S]*?)<\/t[hd]>/gi;
    let cm;
    while ((cm = cellRe.exec(rm[1])) !== null) {
      cells.push(clean(cm[1]).replace(/\|/g, "\\|"));
    }
    rows.push(`| ${cells.join(" | ")} |`);
  }
  if (rows.length === 0) return "";
  const cols = (rows[0].match(/\|/g) || []).length - 1;
  rows.splice(1, 0, `| ${Array(cols).fill("---").join(" | ")} |`);
  return "\n" + rows.join("\n") + "\n";
}

function clean(html) {
  return html
    .replace(/<[^>]+>/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

function decodeEntities(str) {
  return str
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&apos;/g, "'")
    .replace(/&nbsp;/g, " ")
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
    .replace(/&#x([0-9a-f]+);/gi, (_, h) =>
      String.fromCharCode(parseInt(h, 16)),
    );
}

curl -H "Accept: text/markdown" https://YOUR-DOMAIN/