# Robots.txt file from http://www.robottext.co.uk # # Built from information taken from http://www.searchengineworld.com & http://info.webcrawler.com/mak/projects/robots/active/all.txt # # User-agent: Mediapartners-Google Disallow: /cgi-bin/ Disallow: /mailbox/ User-agent: Googlebot Disallow: /cgi-bin/ Disallow: /mailbox/ User-agent: MSNbot Disallow: /cgi-bin/ Disallow: /mailbox/ # Yahoo and Inktomi both use Slurp User-agent: Slurp Disallow: /cgi-bin/ Disallow: /mailbox/ User-agent: WISENutBot Disallow: /cgi-bin/ Disallow: /mailbox/ User-agent: ASKJeeves Disallow: /cgi-bin/ Disallow: /mailbox/ User-agent: ia_archiver Disallow: /cgi-bin/ Disallow: /mailbox/