# ============================================
# ROBOTS.TXT - SEO/AEO/GEO OPTIMIZED
# ============================================
# This file guides search engine crawlers and AI bots
# Last Updated: 2026-02-08
# ============================================

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp (Yahoo)
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Allow AI Crawlers (GEO Optimization)
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Allow Social Media Crawlers
User-agent: FacebookBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Allow SEO Tools
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MozBot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

# ============================================
# CRAWL RATE OPTIMIZATION
# ============================================
# Control crawl frequency to manage server load

Crawl-delay: 1

# ============================================
# SITEMAP DECLARATION
# ============================================
# Point search engines to your sitemaps

Sitemap: https://yourwebsite.com/sitemap.xml
Sitemap: https://yourwebsite.com/sitemap-image.xml
Sitemap: https://yourwebsite.com/sitemap-video.xml
Sitemap: https://yourwebsite.com/sitemap-news.xml (if applicable)

# ============================================
# DISALLOW PATTERNS
# ============================================
# Block access to non-essential or duplicate content

# Admin and backend areas
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /dashboard/
Disallow: /account/
Disallow: /user/

# Search and filter results (duplicate content)
Disallow: /search/
Disallow: /search?
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*?order=

# Cart and checkout (e-commerce)
Disallow: /cart/
Disallow: /checkout/
Disallow: /basket/
Disallow: /add-to-cart/

# User-generated content that may be low quality
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /feed/
Disallow: /rss/

# Development and staging
Disallow: /staging/
Disallow: /test/
Disallow: /dev/
Disallow: /beta/

# Internal tools and utilities
Disallow: /api/
Disallow: /internal/
Disallow: /private/
Disallow: /temp/
Disallow: /tmp/

# Error pages
Disallow: /404/
Disallow: /500/
Disallow: /error/

# Thank you/confirmation pages (after form submission)
Disallow: /thank-you/
Disallow: /confirmation/
Disallow: /success/

# Print versions
Disallow: /print/
Disallow: /*?print=1

# Pagination parameters (optional - depends on your SEO strategy)
# Disallow: /*?page=
# Disallow: /*?p=

# Session and tracking parameters
Disallow: /*?sessionid=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=

# ============================================
# ALLOW SPECIFIC PATTERNS
# ============================================
# Ensure important content is accessible

Allow: /wp-content/uploads/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.pdf$

# ============================================
# HOST DIRECTIVE (Optional)
# ============================================
# Specify your preferred domain version

Host: https://yourwebsite.com

# ============================================
# END OF ROBOTS.TXT
# ============================================
