User-agent: *
Allow: /


User-Agent: MJ12bot
Crawl-Delay: 20

User-agent: Xenu Link Sleuth
Disallow: /

User-agent: LinkCheck by Siteimprove.com
Disallow: /