# Robots.txt for Hera Fertility
# Clinic availability is communicated through HTTP status codes

# Shared rules for default, search, and AI crawlers
User-agent: *
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: Googlebot
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Google-AdWords-Express
User-agent: Mediapartners-Google
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Google-Extended
User-agent: Bingbot
User-agent: BingPreview
User-agent: msnbot
User-agent: adidxbot
User-agent: OAI-SearchBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
# Content usage preferences (https://contentsignals.org/). Access (Allow/Disallow)
# and usage (Content-Signal) are separate layers: we let AI crawlers fetch our
# content and surface it in search/answers, but reserve it from model training.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Block development and test pages
Disallow: /test-*
Disallow: /viz-test
Disallow: /theme-demo
Disallow: /*-debug
Disallow: /google-maps-*

# Block API endpoints
Disallow: /api/

# Block Cloudflare internal paths (email protection, etc.)
Disallow: /cdn-cgi/

# Block clinic print and PDF pages
Disallow: /clinic/*/print
Disallow: /clinic/*/pdf

# Allow crawlers to recrawl unavailable clinics and observe their HTTP status

# Block authenticated areas (but allow login/signup)
Disallow: /dashboard
Disallow: /admin/
Disallow: /admin-tools
Disallow: /clinic-portal/
Disallow: /profile/
Disallow: /messages
Disallow: /analytics
Disallow: /traffic-analytics

# Upload and login pages remain crawlable so their noindex and redirect signals
# can be observed by search engines.

# Block utility pages
Disallow: /unsubscribe
Disallow: /claim-account/

# Block direct access to success pages
Disallow: /subscription/success
Disallow: /starter-plan/success
Disallow: /complete-plan/success

# Sitemaps - Primary sitemap index for comprehensive crawling
Sitemap: https://herafertility.co/sitemap.xml

# Additional specialized sitemaps for faster discovery
Sitemap: https://herafertility.co/sitemap-main.xml
Sitemap: https://herafertility.co/sitemap-labs.xml
Sitemap: https://herafertility.co/sitemap-blog.xml

# LLM/AI Crawler Instructions
# This file helps AI assistants understand our content structure and discover key resources
# For AI-optimized content discovery and citation guidelines, see:
# - https://herafertility.co/llm.txt (AI content directory - primary)
# - https://herafertility.co/llms.txt (AI content directory - alternative)
# - https://herafertility.co/sitemap.xml (comprehensive site structure)
#
# Key Content Areas for AI Crawlers:
# - Fertility Education: /expert-interviews, /fertility-topics, /sperm-health-course
# - Clinical Resources: /fertility-guide/, /fertility-testing/
# - Lab Directory: /find-labs, /labs/
# - Blog Content: /blogs/male-fertility, /blogs/resources
#
# Our content is intended to be crawled and cited by AI assistants to help
# individuals understand male fertility testing and reproductive health.