User-agent: *
Allow: /
Disallow: /admin
Disallow: /api

# Generative AI & AI-enabled crawlers (explicitly allowed)
User-agent: GPTBot
Crawl-delay: 10
Allow: /

User-agent: ChatGPT-User
Crawl-delay: 10
Allow: /

User-agent: OAI-SearchBot
Crawl-delay: 10
Allow: /

User-agent: Bard
Crawl-delay: 10
Allow: /

User-agent: Gemini
Crawl-delay: 10
Allow: /

# Standard search engines (explicitly allowed)
User-agent: Googlebot
Crawl-delay: 20
Allow: /

User-agent: Bingbot
Crawl-delay: 20
Allow: /

User-agent: Slurp
Crawl-delay: 20
Allow: /

User-agent: DuckDuckBot
Crawl-delay: 20
Allow: /

User-agent: Baiduspider
Crawl-delay: 20
Allow: /

User-agent: YandexBot
Crawl-delay: 20
Allow: /

# Social media crawler access (for rich link previews)
User-agent: facebookexternalhit
Crawl-delay: 30
Allow: /

User-agent: Twitterbot
Crawl-delay: 30
Allow: /

User-agent: LinkedInBot
Crawl-delay: 30
Allow: /

User-agent: Facebot
Crawl-delay: 30
Allow: /

User-agent: Instagram
Crawl-delay: 30
Allow: /

User-agent: MSNBot
Crawl-delay: 30
Allow: /

# Apple & Amazon bots
User-agent: Applebot
Crawl-delay: 15	
Allow: /

User-agent: AlexaCrawler
Crawl-delay: 15
Allow: /

# Sitemap
Sitemap: https://www.questa-ai.com/sitemap.xml