# disallow all 
User-agent: *
Disallow: /
Crawl-delay: 10

# but allow only important bots
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: msnbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
User-agent: LinkedInBot
User-agent: facebookexternalhit
User-agent: ia_archiver

# Directories
Allow: /
Disallow: /content/search
Disallow: /Content/Search
Disallow: /content/advancedsearch
Disallow: /Content/Advancedsearch
Disallow: /content/tipafriend
Disallow: /Content/Tipafriend
Disallow: /layout/set/print
Disallow: /Layout/Set/Print
Disallow: /rss
Disallow: /Rss
Disallow: /media
Disallow: /Media
Disallow: /ezinfo
Disallow: /Ezinfo
Disallow: /user/register
Disallow: /User/Register
Disallow: /test-area
Disallow: /Test-Area
Crawl-delay: 10