# AI/search crawlers - explicitly welcomed to public marketing content
# OpenAI crawler docs: https://openai.com/bot
# Perplexity crawler docs: https://docs.perplexity.ai/docs/resources/perplexity-crawlers
# Anthropic crawler docs: https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
# If you run a WAF (Cloudflare/AWS/etc), allow official bot IP ranges:
# - OAI-SearchBot: https://openai.com/searchbot.json
# - GPTBot: https://openai.com/gptbot.json
# - ChatGPT-User: https://openai.com/chatgpt-user.json
# - PerplexityBot: https://www.perplexity.com/perplexitybot.json
# - Perplexity-User: https://www.perplexity.com/perplexity-user.json
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

# Brave Search follows Googlebot-compatible crawl rules, so keep public
# marketing content open here while private app routes stay blocked.
User-agent: Googlebot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Claude-User
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Perplexity-User
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Applebot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: Bytespider
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

User-agent: meta-externalagent
Allow: /
Allow: /llms.txt
Allow: /sitemap.xml
Disallow: /dashboard
Disallow: /api
Disallow: /nova

# Fallback policy for all other crawlers
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /api
Disallow: /nova

Sitemap: https://answeringagent.com/sitemap.xml
# llms.txt is available at: https://answeringagent.com/llms.txt