# =============================
# AGGRESSIVE ROBOTS.TXT FOR DIGITALBURG
# Blocks bad bots, keeps main search engines
# =============================

# Block sensitive areas
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?*

# Block known bad bots and GPT-style scrapers
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: OpenAI
Disallow: /

User-agent: ChatGPT
Disallow: /

# Polite crawling delay for remaining bots
User-agent: *
Crawl-delay: 10