# Estate Lookup Interiors - AI & LLM Optimized robots.txt
# Optimized for maximum AI/LLM crawler visibility and indexing
# Upload to: public_html/robots.txt

# =================================================================
# AI/LLM CRAWLERS - PRIORITY ACCESS (Top 3 + All Major AI Bots)
# =================================================================

# OpenAI (ChatGPT, GPT-4, SearchGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 1

# Anthropic (Claude AI)
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# Google AI (Gemini, Bard, Google AI)
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
Allow: /
Crawl-delay: 1

# Additional AI/LLM Crawlers
User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: facebot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

# =================================================================
# TRADITIONAL SEARCH ENGINE CRAWLERS
# =================================================================

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# =================================================================
# GLOBAL RULES FOR ALL OTHER BOTS
# =================================================================

User-agent: *
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /cgi-bin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: */feed/
Disallow: */trackback/
Disallow: */comments/
Disallow: */xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: *?replytocom=

# =================================================================
# SITEMAPS - CRITICAL FOR AI CRAWLER DISCOVERY
# =================================================================

Sitemap: https://estatelookupinteriors.in/sitemap.xml
Sitemap: https://estatelookupinteriors.in/sitemap_index.xml
Sitemap: https://estatelookupinteriors.in/post-sitemap.xml
Sitemap: https://estatelookupinteriors.in/page-sitemap.xml
Sitemap: https://estatelookupinteriors.in/portfolio-sitemap.xml

# =================================================================
# AI OPTIMIZATION NOTES
# =================================================================
# This robots.txt is optimized for:
# 1. ChatGPT/GPT-4/SearchGPT (OpenAI)
# 2. Claude AI (Anthropic)
# 3. Gemini/Bard/Google AI (Google)
# Plus all other major AI/LLM crawlers for maximum visibility
# Last updated: 2026-02-03
# =================================================================