User-agent: *
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /category/*/feed/
Disallow: /tag/*/feed/
Disallow: /author/

# Allow major AI crawlers
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://www.techshu.com/sitemap_index.xml