# DosePaw — Crawler Policy
# Only index the canonical (English) locale. Block duplicate locale variants
# to prevent 11x crawl amplification and conserve Edge Request budget.

User-agent: *
Allow: /en/
Allow: /en$
Allow: /share/
Allow: /emergency/
Allow: /care/
Disallow: /es/
Disallow: /de/
Disallow: /fr/
Disallow: /it/
Disallow: /ja/
Disallow: /ko/
Disallow: /nl/
Disallow: /pt/
Disallow: /tr/
Disallow: /zh/
Disallow: /api/
Disallow: /_next/
Disallow: /serwist/

# Aggressive crawlers — block entirely (SEO tools, not search engines)
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Rate limit for well-behaved bots
User-agent: Googlebot
Crawl-delay: 2

User-agent: Bingbot
Crawl-delay: 2

# Sitemap (canonical locale only)
Sitemap: https://www.dosepaw.com/sitemap.xml