# Cosmo Clinic Israel - robots.txt
# Updated: 2026-05-28

# ============================================================
# Search Engine Bots — Allowed
# ============================================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

# ============================================================
# AI Search Crawlers — Explicitly Allowed
# We WANT to appear in ChatGPT, Claude, Perplexity, Google AI Overviews
# ============================================================

# OpenAI
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Google — AI Overviews / Gemini
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# ============================================================
# AI Crawlers — Blocked (training-only, no citation value)
# ============================================================

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Amazonbot
Disallow: /

# ============================================================
# Default — All Other Bots
# ============================================================

User-agent: *
Allow: /
# Private areas — NEVER index (owner: Yuval, 2026-06-02)
Disallow: /portal
Disallow: /portal/
Disallow: /portal.html
Disallow: /client-area
Disallow: /client-area/
Disallow: /my-account
Disallow: /dashboard
Disallow: /login
Disallow: /admin
Disallow: /admin/
Disallow: /_docs/
Disallow: /thank-you

# Also block by name in case crawlers ignore wildcards
User-agent: Googlebot
Disallow: /portal
Disallow: /portal/
Disallow: /portal.html
Disallow: /client-area
Disallow: /client-area/
Disallow: /my-account
Disallow: /dashboard
Disallow: /admin
Disallow: /_docs/
Disallow: /thank-you

User-agent: Bingbot
Disallow: /portal
Disallow: /portal/
Disallow: /portal.html
Disallow: /client-area
Disallow: /admin
Disallow: /_docs/

# ============================================================
# Sitemaps
# ============================================================

Sitemap: https://www.cosmoclinic.co.il/sitemap.xml