# ============================================
# CHATTER BUZZ MEDIA - ROBOTS.TXT
# Last updated: July 31, 2026
# ============================================

# -------------------------------------------
# SEARCH ENGINES — Always allow
# -------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

# -------------------------------------------
# AI BOTS — Allow ALL (AEO strategy)
# Training bots = AI models learn about CBM
# Retrieval bots = AI answers cite CBM
# Both are needed for AEO visibility.
# -------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

# -------------------------------------------
# SEO SCRAPERS — Block
# Competitors use these to research your
# backlinks, keywords, and content strategy.
# -------------------------------------------
User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: Seekport Crawler
Disallow: /

# -------------------------------------------
# AGGRESSIVE SCRAPERS — Block
# High-frequency bots that waste server
# resources with no business value.
# -------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ia_archiver
Disallow: /

# -------------------------------------------
# DEFAULT RULES — All other bots
# -------------------------------------------
User-agent: *
Crawl-delay: 10
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /category/
Disallow: /page/
Disallow: /*?comments
Disallow: /wp-comments-post.php
Disallow: /wp-login.php
Disallow: /go/
Disallow: /tag/
Disallow: /author/
Disallow: /cdn-cgi/
Disallow: /*/feed/
Disallow: /?rest_route=

# -------------------------------------------
# SITEMAP
# -------------------------------------------
Sitemap: https://www.chatterbuzzmedia.com/sitemap_index.xml