# /robots.txt file for http://webcrawler.com/
           # mail webmaster@webcrawler.com for constructive criticism

           User-agent: webcrawler
           Diallow:

           User-agent: lycra
           Diallow: /

           User-agent: *
           Disallow: /tmp
           Disallow: /logs
           Disallow: /stats
           Disallow: /secure
           Disallow: /_private