# ============================
# robots.txt
# Last updated: 2025-10-06
# Purpose: Allow AI crawlers & SEO bots, block spammy ones
# ============================

# --- AI Crawlers ---
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# --- Search Engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Yandex
Disallow: /

# --- Block Spammy / Aggressive Crawlers ---
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: *
Disallow: /*?products/

User-agent: PetalBot
Disallow: /

# --- Disallow Sensitive Areas ---
User-agent: *
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /storage/
Disallow: /vendor/
Disallow: /cgi-bin/
Allow: /