# Allow all crawlers including AI crawlers
User-agent: *
Allow: /

# Explicitly allow AI crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Applebot
Allow: /

# Sitemap location
Sitemap: https://www.cheaterscanner.com/sitemap.xml

# Prevent crawling of next.js specific files
Disallow: /_next/static/
Disallow: /_next/image/
Disallow: /.next/
Disallow: /api/

# Block subscription page
Disallow: /subscription

# Allow crawling of static assets
Allow: /static/
Allow: /images/
Allow: /media/
Allow: /assets/

# Block common development paths
Disallow: /dev/
Disallow: /staging/
Disallow: /test/