# agentic-seo-system.com
# Everything is open. This site exists to be read by machines.
#
# The two exceptions are /share/ and /a/ — delivery links carrying a named
# prospect's own document. They are noindex,nofollow at the page level too;
# this file is the belt to that pair of braces. The disallow is repeated in
# every group on purpose: a crawler that matches its own user-agent group
# ignores the * group entirely.
#
# NOTE FOR DEPLOY: Cloudflare changes AI-crawler defaults on 2026-09-15
# (Training + Agent categories blocked by default for new sites and free
# accounts). robots.txt does NOT override that — it is enforced at the edge.
# Set Security > Bots > AI crawler controls to ALLOW Search + Agent + Training
# before launch, then verify with log analysis.

User-agent: *
Allow: /
Disallow: /share/
Disallow: /a/

# Named explicitly so intent is unambiguous to operators reading this file.
User-agent: GPTBot
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: OAI-SearchBot
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: ChatGPT-User
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: PerplexityBot
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: ClaudeBot
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: Claude-SearchBot
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: Google-Extended
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: Applebot-Extended
Allow: /
Disallow: /share/
Disallow: /a/

User-agent: CCBot
Allow: /
Disallow: /share/
Disallow: /a/

Sitemap: https://agentic-seo-system.com/sitemap-index.xml