# Robots.txt for Coastal Haven Real Estate
# https://www.coastalhavensamplesite.com

# Allow all crawlers full access to public content
User-agent: *
Allow: /
Allow: /listings
Allow: /rentals
Allow: /property/
Allow: /rental/
Allow: /sold-property/
Allow: /suburbs/
Allow: /team/
Allow: /journal/
Allow: /contact
Allow: /about
Allow: /sell
Allow: /appraisal

# Crawler-friendly static HTML pages (no JS required)
Allow: /crawl.html
Allow: /rentals.html
Allow: /sold-data.html
Allow: /rental-*.html
Allow: /sold-*.html
Allow: /static-test.html

# Direct index.html URLs (bypass rewrites)
Allow: /listings/index.html
Allow: /rentals/index.html
Allow: /property/*/index.html
Allow: /rental/*/index.html
Allow: /sold-property/*/index.html
Allow: /suburbs/*/index.html

# Crawler-friendly data pages
Allow: /rentals-data
Allow: /sold-data

# Block admin and internal routes only
Disallow: /seo-check

# Google crawlers
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Social media crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

# AI crawlers - explicitly allowed for AI-readability
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: YouBot
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: omgili
Allow: /

# Sitemap locations (multiple formats)
Sitemap: https://www.coastalhavensamplesite.com/sitemap.xml
Sitemap: https://www.coastalhavensamplesite.com/sitemap.txt

# Host directive
Host: https://www.coastalhavensamplesite.com

# Crawler hub for complete static HTML discovery
# Start here: https://www.coastalhavensamplesite.com/crawl.html