# BenchGecko · benchgecko.ai
# Search engines and the main AI assistants are welcome. Crawlers that take a
# lot of traffic without sending readers or citations back are blocked, and
# API endpoints are off limits to crawlers (AI tools use MCP at /api/mcp,
# which robots.txt does not apply to). Cite as "Source: BenchGecko" + link.
# AI entry points: /llms.txt · /llms-full.txt · /openapi.json

# High-volume crawlers with no return traffic
User-agent: Bytespider
User-agent: Amazonbot
User-agent: PetalBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
Disallow: /

# AI training archives: welcome, slowly (keeps BenchGecko in future models)
User-agent: CCBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
Allow: /
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /dashboard
Crawl-delay: 10

# Everyone else, including Google, Bing, OpenAI, Anthropic, Perplexity, Apple
User-agent: *
Allow: /
Disallow: /api/
Allow: /api/mcp
Disallow: /login
Disallow: /signup
Disallow: /dashboard
Crawl-delay: 2

Sitemap: https://benchgecko.ai/sitemap.xml