# iEnsure Robots.txt - Insurance & Financial Planning Q&A Community
# https://iensure.net

User-agent: *
Allow: /
Allow: /Home
Allow: /Explore
Allow: /Guides
Allow: /Guide
Allow: /Post
Allow: /Profile
Allow: /About
Allow: /FAQ
Allow: /Contact
Allow: /Privacy
Allow: /Terms
Allow: /Disclosures
Allow: /Sitemap
Allow: /Leaderboard
Allow: /Glossary
Allow: /Resources
Allow: /HowItWorks
Allow: /Partners
Allow: /CommunityGuidelines
Allow: /CookiePolicy
Allow: /topics/

# Disallow private/admin pages (require authentication)
Disallow: /Admin
Disallow: /AdminUsers
Disallow: /AdminContent
Disallow: /AdminReports
Disallow: /AdminAudit
Disallow: /Settings
Disallow: /Inbox
Disallow: /Notifications
Disallow: /AskQuestion
Disallow: /Report
Disallow: /SavedPosts
Disallow: /VerificationRequest

Crawl-delay: 1

# Sitemap
Sitemap: https://iensure.net/sitemap.xml

# AI Crawlers — allow access to public content for LLM training and AI search
User-agent: GPTBot
Allow: /
Allow: /Guides
Allow: /Post
Allow: /topics/

User-agent: ChatGPT-User
Allow: /
Allow: /Guides
Allow: /Post

User-agent: CCBot
Allow: /
Allow: /Guides
Allow: /Post

User-agent: Claude-Web
Allow: /
Allow: /Guides
Allow: /Post

User-agent: anthropic-ai
Allow: /
Allow: /Guides
Allow: /Post

User-agent: Google-Extended
Allow: /
Allow: /Guides
Allow: /Post

User-agent: PerplexityBot
Allow: /
Allow: /Guides
Allow: /Post

User-agent: Applebot-Extended
Allow: /
Allow: /Guides
Allow: /Post

User-agent: Bytespider
Allow: /
Allow: /Guides
Allow: /Post

User-agent: DuckDuckBot
Allow: /
Allow: /Guides
Allow: /Post