# ============================================ # Maximem.ai - Robots.txt (SEO Optimized) # Last Updated: 2026-08-01 # ============================================ # # POLICY # ------ # Maximem allows all reputable crawlers — traditional search, AI training, and # inference-time retrieval — across the whole public site. Only genuinely # internal surfaces are withheld (see the Disallow block repeated per group). # # A crawler is blocked only when it is undocumented by its vendor, ignores # robots.txt in practice, or cannot be attributed to an identifiable operator. # Bytespider is the only agent that currently meets that bar. Any future block # must be recorded here, with the reason, in the group itself. # # USAGE SIGNALS # ------------- # Two vocabularies are declared on every allowed group. They say the same thing; # they exist because parsers are mid-migration between them. # # Content-Signal (Cloudflare / draft-romm-aipref-contentsignals): # search=yes — allow indexing for traditional search # ai-train=yes — allow use of content for AI model training # ai-input=yes — allow inference-time retrieval (RAG, citations) # # Content-Usage (IETF AIPREF — draft-ietf-aipref-attach / -vocab): # train-ai=y — allow use in producing or refining a generative model # search=y — allow use in applications that direct users to the source # # BOT TIERS # --------- # Most vendors run separate agents for training, for search indexing, and for # fetching a page because a user just asked a question. They are independent: # allowing ClaudeBot does nothing for Claude-User or Claude-SearchBot. The # user-initiated tier is what produces citations, so all three are listed. # # MAINTENANCE # ----------- # robots.txt does not inherit rules from User-agent: * once a named agent has # its own group. Every named group therefore repeats the full Disallow block. # When adding a new named agent, copy an existing group verbatim and change # only the User-agent line. User-agent: * Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── OpenAI ────────────────────────────────────────────────────────────── # GPTBot = training · OAI-SearchBot = search index · ChatGPT-User = user-initiated User-agent: GPTBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: OAI-SearchBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: ChatGPT-User Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Anthropic ─────────────────────────────────────────────────────────── # ClaudeBot = training · Claude-SearchBot = search index · Claude-User = user-initiated # anthropic-ai is the legacy token, retained for older crawler builds. User-agent: ClaudeBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Claude-SearchBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Claude-User Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: anthropic-ai Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Google ────────────────────────────────────────────────────────────── # Google-Extended is an opt-out token, not a crawler; Google-CloudVertexBot # and Google-NotebookLM are real fetchers. User-agent: Google-Extended Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Google-CloudVertexBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Google-NotebookLM Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Microsoft ─────────────────────────────────────────────────────────── # Bingbot is the crawler behind Copilot. It inherited User-agent: * before # this block, which allowed it but left it the one major crawler with no # content signal of its own. User-agent: Bingbot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Perplexity ────────────────────────────────────────────────────────── # PerplexityBot = index · Perplexity-User = user-initiated retrieval User-agent: PerplexityBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Perplexity-User Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Apple ─────────────────────────────────────────────────────────────── # Applebot is the crawler; Applebot-Extended is the AI-training opt-out token. User-agent: Applebot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Applebot-Extended Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Meta ──────────────────────────────────────────────────────────────── # meta-externalagent = training/indexing · meta-externalfetcher = user-initiated User-agent: meta-externalagent Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: meta-externalfetcher Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Others ────────────────────────────────────────────────────────────── User-agent: MistralAI-User Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: DuckAssistBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: Amazonbot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: CCBot Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share User-agent: cohere-ai Content-Signal: search=yes, ai-train=yes, ai-input=yes Content-Usage: train-ai=y, search=y Allow: / Disallow: /admin Disallow: /echo/ Disallow: /uninstall-feedback Disallow: /font-compare Disallow: /wrapped/share # ── Blocked ───────────────────────────────────────────────────────────── # Bytespider (ByteDance): undocumented by its vendor and widely reported to # ignore robots.txt. Blocked until ByteDance publishes crawler documentation. User-agent: Bytespider Disallow: / # Sitemap Locations Sitemap: https://www.maximem.ai/sitemap.xml Sitemap: https://www.maximem.ai/sitemap-pages.xml Sitemap: https://www.maximem.ai/sitemap-blog.xml Sitemap: https://www.maximem.ai/sitemap-images.xml Sitemap: https://www.maximem.ai/sitemap-glossary.xml Sitemap: https://www.maximem.ai/sitemap-videos.xml Sitemap: https://docs.maximem.ai/sitemap.xml