# robots.txt for hussainwh.info

User-agent: *
Allow: /
Disallow: /auth/
Disallow: /api/ # Disallow crawling of Next.js API routes including the proxy
Crawl-delay: 5 # Be kind to servers, though major bots may ignore this

# Specific instructions for Googlebot
User-agent: Googlebot
Allow: /
Disallow: /auth/
Disallow: /api/
# Crawl-delay: 2 # Google often ignores crawl-delay but you can set it

# Specific instructions for Bingbot
User-agent: Bingbot
Allow: /
Disallow: /auth/
Disallow: /api/
Crawl-delay: 5

# Specific instructions for other known good bots
User-agent: DuckDuckBot
Allow: /
Disallow: /auth/
Disallow: /api/

User-agent: Baiduspider
Allow: /
Disallow: /auth/
Disallow: /api/

User-agent: Yandex
Allow: /
Disallow: /auth/
Disallow: /api/

# Sitemap location - IMPORTANT: Update this to your actual production domain
Sitemap: https://hussainwh.info/sitemap.xml