# =============================================================================
# robots.txt for Citely (https://citely.ai)
# AI Citation Checker & Source Finder
# =============================================================================

# -----------------------------------------------------------------------------
# Tier 1: AI crawlers — ALLOW (high-value search & chat surfaces)
# OpenAI ChatGPT search, search-only bot, user-initiated browsing
# Anthropic Claude, Perplexity AI
# -----------------------------------------------------------------------------

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*/admin/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /pay-success/
Disallow: /*/pay-success/
Disallow: /search

# -----------------------------------------------------------------------------
# Tier 2: AI crawlers — ALLOW (secondary but valuable channels)
# Google Gemini, Google other AI, Apple Intelligence / Siri
# Amazon Alexa / shopping, Meta AI
# -----------------------------------------------------------------------------

User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*/admin/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /pay-success/
Disallow: /*/pay-success/
Disallow: /search

# -----------------------------------------------------------------------------
# Tier 3: Blocked crawlers (aggressive scraping / training-only, low value)
# -----------------------------------------------------------------------------

# ByteDance — aggressive crawling, low referral value
User-agent: Bytespider
Disallow: /

# Common Crawl — training data only, no search surface
User-agent: CCBot
Disallow: /

# -----------------------------------------------------------------------------
# Default rules for all other crawlers (including Googlebot, Bingbot, etc.)
# -----------------------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*/admin/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /pay-success/
Disallow: /*/pay-success/
Disallow: /search

# -----------------------------------------------------------------------------
# Sitemap
# -----------------------------------------------------------------------------

Sitemap: https://citely.ai/sitemap.xml