# robots.txt for models-aggregator
# Per ai-visibility-geo.md §1.2 — block training crawlers, allow search/retrieval

# ============================================================
# Search & retrieval crawlers: ALLOW
# ============================================================
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# ============================================================
# User-triggered fetches (when a user shares your URL with the AI): ALLOW
# ============================================================
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

# ============================================================
# Standard search engines: ALLOW
# ============================================================
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ============================================================
# Training crawlers: BLOCK
# ============================================================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# ============================================================
# Opt-out tokens (Google + Apple): TELL THEM NO
# ============================================================
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# ============================================================
# Undeclared / low-rule-following: BLOCK
# ============================================================
User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

# Sitemaps
Sitemap: https://modelrover.com/sitemap.xml
Sitemap: https://modelrover.com/sitemap-models.xml
Sitemap: https://modelrover.com/sitemap-pages.xml