# robots.txt for crunchable.net
User-agent: *
Disallow: /exs/        #contains unfinished pages
Disallow: /pageparts/  #fragments of pages
Disallow: /errpgs/     #error pages and related

User-agent: Slurp
Crawl-delay: 5