# robots.txt for RealEstate.EG
# Updated: 2026-04-29
# AI crawlers explicitly welcomed for content discovery

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /app/
Disallow: /bootstrap/
Disallow: /config/
Disallow: /database/
Disallow: /resources/
Disallow: /routes/
Disallow: /storage/
Disallow: /temp/
Disallow: /tests/
Disallow: /vendor/
Disallow: /search/where-live
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=
Disallow: /*&utm_*
Disallow: /*?_token=
Disallow: /*/login
Disallow: /*/logout
Disallow: /add-listing/csrf-refresh
Disallow: /admin/leads/
Disallow: /*/contact-us/save-form

# === AI Training Crawlers (welcomed) ===
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: cohere-ai
Allow: /

# === AI Search/Retrieval Crawlers ===
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# === Block bad actors ===
User-agent: Bytespider
Disallow: /

# Sitemap index
Sitemap: https://realestate.eg/sitemap.index.xml
Sitemap: https://realestate.eg/sitemap.listing_ar_1.xml
Sitemap: https://realestate.eg/sitemap.listing_ar_2.xml
Sitemap: https://realestate.eg/sitemap.listing_ar_3.xml
Sitemap: https://realestate.eg/sitemap.listing_en_1.xml
Sitemap: https://realestate.eg/sitemap.listing_en_2.xml
Sitemap: https://realestate.eg/sitemap.listing_en_3.xml
Sitemap: https://realestate.eg/sitemap.developer_1.xml
Sitemap: https://realestate.eg/sitemap.for_sale.xml
Sitemap: https://realestate.eg/sitemap.blog.xml

# AI-discoverable summary
# https://realestate.eg/llms.txt

# Bing crawler — welcome with maximum crawl rate
User-agent: bingbot
Allow: /
Crawl-delay: 1

# IndexNow key
# Key: d1ac02817cb27b2a541e45d812a06d6e
# Verification: https://realestate.eg/d1ac02817cb27b2a541e45d812a06d6e.txt