# $Id: robots.txt,v 1.1 2008/10/30 12:47:08 don Exp $ # robots.txt generated at http://www.mcanerin.com #User-agent: Googlebot #Disallow: #User-agent: Googlebot-Image #Disallow: #User-agent: MSNBot #Disallow: #User-agent: Slurp #Disallow: #User-agent: Teoma #Disallow: #User-agent: twiceler #Disallow: #User-agent: Gigabot #Disallow: #User-agent: Scrubby #Disallow: #User-agent: Robozilla #Disallow: #User-agent: Nutch #Disallow: #User-agent: ia_archiver #Disallow: #User-agent: baiduspider #Disallow: #User-agent: yahoo-mmcrawler #Disallow: #User-agent: psbot #Disallow: #User-agent: asterias #Disallow: #User-agent: yahoo-blogs/v3.9 #Disallow: User-agent: * Disallow: /includes/ Crawl-delay: 20 #Sitemap: http://www.yoursite.com/sitemap.gz