# robots.txt for InstantIndexNow
# https://instantindexnow.com/robots.txt

# Sitemap location
Sitemap: https://instantindexnow.com/sitemap.xml

# Default rules - allow all bots
User-agent: *
Allow: /

# Allow all content for search engines
Allow: /about
Allow: /faq
Allow: /api/*

# AI Search Engine Bots - Explicitly allowed for GEO optimization
# Googlebot (Google Search & AI Overview)
User-agent: Googlebot
Allow: /

# Bingbot (Bing Search & Microsoft Copilot)
User-agent: Bingbot
Allow: /

# PerplexityBot (Perplexity AI)
User-agent: PerplexityBot
Allow: /

# ChatGPT-User (ChatGPT with browsing capability)
User-agent: ChatGPT-User
Allow: /

# GPTBot (OpenAI's crawler)
User-agent: GPTBot
Allow: /

# ClaudeBot / anthropic-ai (Anthropic's Claude AI)
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Bytespider (ByteDance/Douyin)
User-agent: Bytespider
Allow: /

# Yandex bot
User-agent: Yandex
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# Disallow admin/private areas (if any in future)
# Disallow: /admin/
# Disallow: /private/

# Crawl delay (optional - remove for aggressive crawling)
# Crawl-delay: 1

# Common bot exclusions (security)
Disallow: /*?*sessionid=
Disallow: /*?*utm_*
Disallow: /*.json$