# AI bot policy: retrieval/citation bots ALLOWED, model-training crawlers DISALLOWED.
# NOTE: must be paired with the Cloudflare bot-management change (Security > Bots) —
# the WAF currently 403s these agents regardless of robots.txt.

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

# --- Answer-engine / retrieval / citation / user-fetch bots: ALLOW ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Applebot
Allow: /

# --- Model-training crawlers: DISALLOW (ai-train=no) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: cohere-ai
Disallow: /

Sitemap: https://tarabos.ai/sitemap.xml