# YOYAKU.IO Robots.txt - Optimized for Music Domain 2025-08-24
# Updated 2025-10-15: Block WooCommerce AJAX endpoint (Cloudways recommendation)

# ALLOWED BOTS - Useful for music domain
User-agent: Googlebot
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Bingbot
Crawl-delay: 10
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OpenAI-SearchBot
Allow: /

User-agent: MusicBot
Allow: /

User-agent: DiscogsCrawler
Allow: /

User-agent: BandcampBot
Allow: /

# Social Media Bots - Allowed for sharing
User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Crawl-delay: 10
Allow: /

User-agent: TwitterBot
Crawl-delay: 10
Allow: /

# SEO & Analytics Bots - Allowed for insights
User-agent: SemrushBot
Crawl-delay: 20
Allow: /

User-agent: AhrefsBot
Crawl-delay: 20
Allow: /

User-agent: MJ12bot
Crawl-delay: 30
Allow: /

User-agent: DotBot
Crawl-delay: 15
Allow: /

# DEFAULT - All other bots with restrictions
User-agent: *
Allow: /

# Disallow admin and system pages
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Disallow e-commerce pages
Disallow: /checkout/
Disallow: /cart/
Disallow: /myaccount/
Disallow: /*?add-to-cart=*
Disallow: /*?removed_item=*

# Block WooCommerce AJAX endpoints (Cloudways recommendation 2025-10-15)
Disallow: /?wc-ajax=get_refreshed_fragments
Disallow: /*?wc-ajax=get_refreshed_fragments

# Allow important directories
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*/assets/

# BLOCKED BOTS - Only resource intensive without value
User-agent: Bytespider
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: bytespider
Disallow: /

User-agent: BingPreview
Disallow: /

User-agent: TelegramBot
Disallow: /

User-agent: WhatsApp
Disallow: /

# Crawl delays for heavy bots
User-agent: *
Crawl-delay: 5

# Sitemap
Sitemap: https://yoyaku.io/sitemap_index.xml