# Disallow all crawlers access to certain pages.
User-agent: Steeler
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: Shim-Crawler
Disallow: /

User-agent: baiduspider
Disallow: /